{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "![alt text](../img/conve.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pprint\n",
    "import itertools\n",
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append(os.path.dirname(os.getcwd()))\n",
    "from data import WN18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config:\n",
    "    n_epochs = 100\n",
    "    batch_size = 150\n",
    "    embed_dim = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "e: entity\n",
    "s: subject\n",
    "p: predicate\n",
    "o: object\n",
    "\"\"\"\n",
    "def glance_dict(d, n=5):\n",
    "    return dict(itertools.islice(d.items(), n))\n",
    "\n",
    "\n",
    "def read_triples(path):\n",
    "    triples = []\n",
    "    with open(path, 'rt') as f:\n",
    "        for line in f.readlines():\n",
    "            s, p, o = line.split()\n",
    "            triples += [(s.strip(), p.strip(), o.strip())]\n",
    "    return triples\n",
    "\n",
    "\n",
    "def load_triple():\n",
    "    WN18.download()\n",
    "    triples_tr = read_triples('../data/WN18/wn18/train.txt')\n",
    "    triples_va = read_triples('../data/WN18/wn18/valid.txt')\n",
    "    triples_te = read_triples('../data/WN18/wn18/test.txt')\n",
    "    triples_all = triples_tr + triples_va + triples_te\n",
    "    return triples_all, triples_tr, triples_va, triples_te\n",
    "\n",
    "\n",
    "def build_vocab(triples):\n",
    "    params = {}\n",
    "    e_set = {s for (s, p, o) in triples} | {o for (s, p, o) in triples}\n",
    "    p_set = {p for (s, p, o) in triples}\n",
    "    params['e_vocab_size'] = len(e_set)\n",
    "    params['p_vocab_size'] = len(p_set)\n",
    "    e2idx = {e: idx for idx, e in enumerate(sorted(e_set))}\n",
    "    p2idx = {p: idx for idx, p in enumerate(sorted(p_set))}\n",
    "    return e2idx, p2idx, params\n",
    "\n",
    "\n",
    "def build_multi_label(triples_tr):\n",
    "    sp2o = {}\n",
    "    for (_s, _p, _o) in triples_tr:\n",
    "        s, p, o = e2idx[_s], p2idx[_p], e2idx[_o] \n",
    "        if (s,p) not in sp2o:\n",
    "            sp2o[(s,p)] = [o]\n",
    "        else:\n",
    "            if o not in sp2o[(s,p)]:\n",
    "                sp2o[(s,p)].append(o)\n",
    "    return sp2o\n",
    "\n",
    "\n",
    "def get_y(triples_tr, e2idx, p2idx, sp2o):\n",
    "    y = []\n",
    "    for (_s, _p, _o) in triples_tr:\n",
    "        s, p, o = e2idx[_s], p2idx[_p], e2idx[_o] \n",
    "        temp = np.zeros([len(e2idx)])\n",
    "        temp[sp2o[(s,p)]] = 1.\n",
    "        y.append(temp)\n",
    "    y = np.asarray(y)\n",
    "    return y\n",
    "\n",
    "\n",
    "def next_train_batch(triples_tr, e2idx, p2idx, sp2o):\n",
    "    for i in range(0, len(triples_tr), Config.batch_size):\n",
    "        _triples_tr = triples_tr[i: i+Config.batch_size]\n",
    "        x_s = np.asarray([e2idx[s] for (s, p, o) in _triples_tr], dtype=np.int32)\n",
    "        x_p = np.asarray([p2idx[p] for (s, p, o) in _triples_tr], dtype=np.int32)\n",
    "        y = get_y(_triples_tr, e2idx, p2idx, sp2o)\n",
    "        yield (x_s, x_p, y)\n",
    "\n",
    "\n",
    "def train_input_fn(triples_tr, e2idx, p2idx, s2p2o):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: next_train_batch(sklearn.utils.shuffle(triples_tr),\n",
    "                                 e2idx,\n",
    "                                 p2idx,\n",
    "                                 s2p2o),\n",
    "        (tf.int32, tf.int32, tf.float32),\n",
    "        (tf.TensorShape([None]),\n",
    "         tf.TensorShape([None]),\n",
    "         tf.TensorShape([None, len(e2idx)])))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    x_s, x_p, y = iterator.get_next()\n",
    "    return {'s': x_s, 'p': x_p}, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def o_next_batch(eval_triples,\n",
    "                 e2idx, \n",
    "                 p2idx):\n",
    "    for (s, p, o) in tqdm(eval_triples, total=len(eval_triples), ncols=70):\n",
    "        s_idx, p_idx = e2idx[s], p2idx[p]\n",
    "        yield np.atleast_1d(s_idx), np.atleast_1d(p_idx)\n",
    "\n",
    "\n",
    "def o_input_fn(eval_triples,\n",
    "               e2idx, \n",
    "               p2idx):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: o_next_batch(eval_triples,\n",
    "                             e2idx, \n",
    "                             p2idx),\n",
    "        (tf.int32, tf.int32),\n",
    "        (tf.TensorShape([None,]),\n",
    "         tf.TensorShape([None,])))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    s, p = iterator.get_next()\n",
    "    return {'s': s, 'p': p}\n",
    "\n",
    "\n",
    "def evaluate_rank(model,\n",
    "                  valid_triples,\n",
    "                  test_triples,\n",
    "                  all_triples,\n",
    "                  e2idx,\n",
    "                  p2idx):\n",
    "    for eval_name, eval_triples in [('test', test_triples)]:\n",
    "        _scores_o = list(model.predict(\n",
    "            lambda: o_input_fn(eval_triples,\n",
    "                               e2idx, \n",
    "                               p2idx)))\n",
    "        ScoresO = np.reshape(_scores_o, [len(eval_triples), len(e2idx)])\n",
    "        ranks_o, filtered_ranks_o = [], []\n",
    "        for ((s, p, o), scores_o) in tqdm(zip(eval_triples, ScoresO),\n",
    "                                          total=len(eval_triples),\n",
    "                                          ncols=70):\n",
    "            s_idx, p_idx, o_idx = e2idx[s], p2idx[p], e2idx[o]\n",
    "            ranks_o += [1 + np.argsort(np.argsort(- scores_o))[o_idx]]\n",
    "            filtered_scores_o = scores_o.copy()\n",
    "            rm_idx_o = [e2idx[fo] for (fs, fp, fo) in all_triples if fs == s and fp == p and fo != o]\n",
    "            filtered_scores_o[rm_idx_o] = - np.inf\n",
    "            filtered_ranks_o += [1 + np.argsort(np.argsort(- filtered_scores_o))[o_idx]]\n",
    "        for setting_name, setting_ranks in [('Raw', ranks_o), ('Filtered', filtered_ranks_o)]:\n",
    "            mean_rank = np.mean(1 / np.asarray(setting_ranks))\n",
    "            print('[{}] {} MRR: {}'.format(eval_name, setting_name, mean_rank))\n",
    "            for k in [1, 3, 5, 10]:\n",
    "                hits_at_k = np.mean(np.asarray(setting_ranks) <= k) * 100\n",
    "                print('[{}] {} Hits@{}: {}'.format(eval_name, setting_name, k, hits_at_k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(features, mode, params):\n",
    "    batch_sz = tf.shape(features['s'])[0]\n",
    "    is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    \n",
    "    e_embed = tf.get_variable('e_embed',\n",
    "                              [params['e_vocab_size'], Config.embed_dim],\n",
    "                              initializer=tf.variance_scaling_initializer())\n",
    "    p_embed = tf.get_variable('p_embed',\n",
    "                              [params['p_vocab_size'], Config.embed_dim],\n",
    "                              initializer=tf.variance_scaling_initializer())\n",
    "    \n",
    "    s = tf.nn.embedding_lookup(e_embed, features['s'])\n",
    "    p = tf.nn.embedding_lookup(p_embed, features['p'])\n",
    "    \n",
    "    x = tf.concat([tf.reshape(s, [batch_sz, 10, 20, 1]),\n",
    "                   tf.reshape(p, [batch_sz, 10, 20, 1]),], axis=1)\n",
    "    x = tf.layers.batch_normalization(x, training=is_training)\n",
    "    x = tf.layers.dropout(x, 0.2, training=is_training)\n",
    "    \n",
    "    x = tf.layers.conv2d(x, 32, (3,3), activation=tf.nn.relu)\n",
    "    x = tf.layers.flatten(x)\n",
    "    x = tf.layers.batch_normalization(x, training=is_training)\n",
    "    x = tf.layers.dropout(x, 0.2, training=is_training)\n",
    "    \n",
    "    x = tf.layers.dense(x, Config.embed_dim, tf.nn.relu)\n",
    "    x = tf.layers.batch_normalization(x, training=is_training)\n",
    "    x = tf.layers.dropout(x, 0.2, training=is_training)\n",
    "    \n",
    "    logits = tf.matmul(x, e_embed, transpose_b=True)\n",
    "    return logits\n",
    "    \n",
    "    \n",
    "def model_fn(features, labels, mode, params):\n",
    "    logits = forward(features, mode, params)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        tf.logging.info('\\n'+pprint.pformat(tf.trainable_variables()))\n",
    "        tf.logging.info('params: %d'%count_train_params())\n",
    "        \n",
    "        global_step = tf.train.get_global_step()\n",
    "        \n",
    "        loss_op = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,\n",
    "                                                                        labels=labels))\n",
    "        \n",
    "        with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):\n",
    "            train_op = tf.train.AdamOptimizer().minimize(loss_op,\n",
    "                                                         global_step = global_step)\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(mode = mode,\n",
    "                                          loss = loss_op,\n",
    "                                          train_op = train_op)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions = tf.sigmoid(logits))\n",
    "\n",
    "\n",
    "def count_train_params():\n",
    "    return np.sum([np.prod([d.value for d in v.get_shape()]) for v in tf.trainable_variables()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files Already Downloaded\n",
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x10aafb710>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Epoch 1\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 4260542.0, step = 1\n",
      "INFO:tensorflow:global_step/sec: 1.99769\n",
      "INFO:tensorflow:loss = 1549495.4, step = 101 (50.059 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92098\n",
      "INFO:tensorflow:loss = 292097.84, step = 201 (52.057 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03088\n",
      "INFO:tensorflow:loss = 136423.56, step = 301 (49.240 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01693\n",
      "INFO:tensorflow:loss = 76119.4, step = 401 (49.580 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0168\n",
      "INFO:tensorflow:loss = 57156.176, step = 501 (49.583 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98012\n",
      "INFO:tensorflow:loss = 45601.996, step = 601 (50.502 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9928\n",
      "INFO:tensorflow:loss = 34136.26, step = 701 (50.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99316\n",
      "INFO:tensorflow:loss = 30990.7, step = 801 (50.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9425\n",
      "INFO:tensorflow:loss = 25633.475, step = 901 (51.480 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 943 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 26725.914.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-943\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 294.63it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:47<00:00, 46.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.012630274747911917\n",
      "[test] Raw Hits@1: 0.5\n",
      "[test] Raw Hits@3: 1.28\n",
      "[test] Raw Hits@5: 1.82\n",
      "[test] Raw Hits@10: 2.44\n",
      "[test] Filtered MRR: 0.012637126006984856\n",
      "[test] Filtered Hits@1: 0.5\n",
      "[test] Filtered Hits@3: 1.28\n",
      "[test] Filtered Hits@5: 1.82\n",
      "[test] Filtered Hits@10: 2.44\n",
      "INFO:tensorflow:Epoch 2\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-943\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 944 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 22629.393, step = 944\n",
      "INFO:tensorflow:global_step/sec: 2.01042\n",
      "INFO:tensorflow:loss = 21919.732, step = 1044 (49.742 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92704\n",
      "INFO:tensorflow:loss = 18878.236, step = 1144 (51.893 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99993\n",
      "INFO:tensorflow:loss = 22403.715, step = 1244 (50.002 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17593\n",
      "INFO:tensorflow:loss = 16173.272, step = 1344 (45.957 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17692\n",
      "INFO:tensorflow:loss = 13576.0205, step = 1444 (45.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18225\n",
      "INFO:tensorflow:loss = 15912.272, step = 1544 (45.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.15093\n",
      "INFO:tensorflow:loss = 20674.135, step = 1644 (46.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.12166\n",
      "INFO:tensorflow:loss = 15246.1455, step = 1744 (47.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17502\n",
      "INFO:tensorflow:loss = 13815.229, step = 1844 (45.976 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1886 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 13391.52.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-1886\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.69it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.01567496903668097\n",
      "[test] Raw Hits@1: 0.8\n",
      "[test] Raw Hits@3: 1.7399999999999998\n",
      "[test] Raw Hits@5: 1.8399999999999999\n",
      "[test] Raw Hits@10: 2.76\n",
      "[test] Filtered MRR: 0.01579012670680101\n",
      "[test] Filtered Hits@1: 0.8\n",
      "[test] Filtered Hits@3: 1.7399999999999998\n",
      "[test] Filtered Hits@5: 1.8399999999999999\n",
      "[test] Filtered Hits@10: 2.82\n",
      "INFO:tensorflow:Epoch 3\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-1886\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1887 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 14796.846, step = 1887\n",
      "INFO:tensorflow:global_step/sec: 2.16085\n",
      "INFO:tensorflow:loss = 11822.436, step = 1987 (46.280 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17347\n",
      "INFO:tensorflow:loss = 16179.06, step = 2087 (46.009 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17337\n",
      "INFO:tensorflow:loss = 12118.318, step = 2187 (46.011 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17964\n",
      "INFO:tensorflow:loss = 12927.409, step = 2287 (45.879 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17565\n",
      "INFO:tensorflow:loss = 9861.291, step = 2387 (45.963 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17865\n",
      "INFO:tensorflow:loss = 9969.696, step = 2487 (45.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18369\n",
      "INFO:tensorflow:loss = 10637.787, step = 2587 (45.794 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18587\n",
      "INFO:tensorflow:loss = 13614.486, step = 2687 (45.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17978\n",
      "INFO:tensorflow:loss = 8436.51, step = 2787 (45.876 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2829 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 10759.6045.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-2829\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 306.71it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.015887729372295985\n",
      "[test] Raw Hits@1: 0.76\n",
      "[test] Raw Hits@3: 1.54\n",
      "[test] Raw Hits@5: 2.36\n",
      "[test] Raw Hits@10: 3.32\n",
      "[test] Filtered MRR: 0.01629064487897077\n",
      "[test] Filtered Hits@1: 0.7799999999999999\n",
      "[test] Filtered Hits@3: 1.5599999999999998\n",
      "[test] Filtered Hits@5: 2.4\n",
      "[test] Filtered Hits@10: 3.4000000000000004\n",
      "INFO:tensorflow:Epoch 4\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-2829\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2830 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 8671.006, step = 2830\n",
      "INFO:tensorflow:global_step/sec: 2.16619\n",
      "INFO:tensorflow:loss = 8911.588, step = 2930 (46.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17505\n",
      "INFO:tensorflow:loss = 7516.597, step = 3030 (45.976 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17533\n",
      "INFO:tensorflow:loss = 7889.7114, step = 3130 (45.971 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17451\n",
      "INFO:tensorflow:loss = 6832.2437, step = 3230 (45.986 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17698\n",
      "INFO:tensorflow:loss = 8739.006, step = 3330 (45.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18036\n",
      "INFO:tensorflow:loss = 7583.8, step = 3430 (45.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17814\n",
      "INFO:tensorflow:loss = 8386.3545, step = 3530 (45.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17515\n",
      "INFO:tensorflow:loss = 7342.1167, step = 3630 (45.974 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.16359\n",
      "INFO:tensorflow:loss = 6960.8525, step = 3730 (46.220 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3772 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 7659.3013.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-3772\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.61it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.024521034789353725\n",
      "[test] Raw Hits@1: 1.52\n",
      "[test] Raw Hits@3: 2.76\n",
      "[test] Raw Hits@5: 3.08\n",
      "[test] Raw Hits@10: 3.7199999999999998\n",
      "[test] Filtered MRR: 0.025502174923392597\n",
      "[test] Filtered Hits@1: 1.6400000000000001\n",
      "[test] Filtered Hits@3: 2.8000000000000003\n",
      "[test] Filtered Hits@5: 3.1199999999999997\n",
      "[test] Filtered Hits@10: 3.82\n",
      "INFO:tensorflow:Epoch 5\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-3772\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3773 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 7390.5547, step = 3773\n",
      "INFO:tensorflow:global_step/sec: 2.14701\n",
      "INFO:tensorflow:loss = 5738.231, step = 3873 (46.578 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.14564\n",
      "INFO:tensorflow:loss = 7596.1343, step = 3973 (46.606 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.15991\n",
      "INFO:tensorflow:loss = 6572.714, step = 4073 (46.298 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.15853\n",
      "INFO:tensorflow:loss = 5389.486, step = 4173 (46.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.14933\n",
      "INFO:tensorflow:loss = 5639.572, step = 4273 (46.526 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.159\n",
      "INFO:tensorflow:loss = 5154.188, step = 4373 (46.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.15382\n",
      "INFO:tensorflow:loss = 6202.2153, step = 4473 (46.429 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.16053\n",
      "INFO:tensorflow:loss = 6095.4287, step = 4573 (46.285 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.15842\n",
      "INFO:tensorflow:loss = 4609.2456, step = 4673 (46.331 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 4715 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 4984.017.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-4715\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.65it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:41<00:00, 49.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.03175189919298796\n",
      "[test] Raw Hits@1: 2.34\n",
      "[test] Raw Hits@3: 3.34\n",
      "[test] Raw Hits@5: 3.64\n",
      "[test] Raw Hits@10: 4.58\n",
      "[test] Filtered MRR: 0.03336203192318166\n",
      "[test] Filtered Hits@1: 2.52\n",
      "[test] Filtered Hits@3: 3.42\n",
      "[test] Filtered Hits@5: 3.82\n",
      "[test] Filtered Hits@10: 4.760000000000001\n",
      "INFO:tensorflow:Epoch 6\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-4715\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 4716 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 4969.667, step = 4716\n",
      "INFO:tensorflow:global_step/sec: 2.1408\n",
      "INFO:tensorflow:loss = 5683.468, step = 4816 (46.713 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.14934\n",
      "INFO:tensorflow:loss = 4544.382, step = 4916 (46.526 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09338\n",
      "INFO:tensorflow:loss = 5282.4404, step = 5016 (47.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00201\n",
      "INFO:tensorflow:loss = 5280.784, step = 5116 (49.950 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06279\n",
      "INFO:tensorflow:loss = 4905.535, step = 5216 (48.478 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06058\n",
      "INFO:tensorflow:loss = 5258.823, step = 5316 (48.530 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03053\n",
      "INFO:tensorflow:loss = 4971.246, step = 5416 (49.249 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0263\n",
      "INFO:tensorflow:loss = 4133.519, step = 5516 (49.351 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03002\n",
      "INFO:tensorflow:loss = 4582.939, step = 5616 (49.260 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5658 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 4695.027.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-5658\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 299.98it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:48<00:00, 46.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.03817646706314746\n",
      "[test] Raw Hits@1: 2.92\n",
      "[test] Raw Hits@3: 3.88\n",
      "[test] Raw Hits@5: 4.46\n",
      "[test] Raw Hits@10: 5.42\n",
      "[test] Filtered MRR: 0.04016390205413305\n",
      "[test] Filtered Hits@1: 3.1199999999999997\n",
      "[test] Filtered Hits@3: 4.04\n",
      "[test] Filtered Hits@5: 4.66\n",
      "[test] Filtered Hits@10: 5.62\n",
      "INFO:tensorflow:Epoch 7\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-5658\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 5659 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 3940.503, step = 5659\n",
      "INFO:tensorflow:global_step/sec: 2.06034\n",
      "INFO:tensorflow:loss = 4904.322, step = 5759 (48.537 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03927\n",
      "INFO:tensorflow:loss = 5211.005, step = 5859 (49.037 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06574\n",
      "INFO:tensorflow:loss = 4728.907, step = 5959 (48.409 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06755\n",
      "INFO:tensorflow:loss = 5102.719, step = 6059 (48.366 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06885\n",
      "INFO:tensorflow:loss = 3967.1965, step = 6159 (48.336 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.11572\n",
      "INFO:tensorflow:loss = 4187.9478, step = 6259 (47.266 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07945\n",
      "INFO:tensorflow:loss = 3740.7737, step = 6359 (48.089 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09587\n",
      "INFO:tensorflow:loss = 3758.4932, step = 6459 (47.713 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04906\n",
      "INFO:tensorflow:loss = 3988.7478, step = 6559 (48.803 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 6601 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 3447.0967.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-6601\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 300.47it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:46<00:00, 47.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.04722178226980461\n",
      "[test] Raw Hits@1: 3.7199999999999998\n",
      "[test] Raw Hits@3: 4.62\n",
      "[test] Raw Hits@5: 5.42\n",
      "[test] Raw Hits@10: 6.4\n",
      "[test] Filtered MRR: 0.0492486679690125\n",
      "[test] Filtered Hits@1: 3.88\n",
      "[test] Filtered Hits@3: 4.82\n",
      "[test] Filtered Hits@5: 5.62\n",
      "[test] Filtered Hits@10: 6.74\n",
      "INFO:tensorflow:Epoch 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-6601\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 6602 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 4125.573, step = 6602\n",
      "INFO:tensorflow:global_step/sec: 2.08507\n",
      "INFO:tensorflow:loss = 3718.433, step = 6702 (47.962 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04804\n",
      "INFO:tensorflow:loss = 3190.4822, step = 6802 (48.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.10693\n",
      "INFO:tensorflow:loss = 3608.4907, step = 6902 (47.463 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06604\n",
      "INFO:tensorflow:loss = 3245.4932, step = 7002 (48.402 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.10436\n",
      "INFO:tensorflow:loss = 3278.821, step = 7102 (47.520 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09935\n",
      "INFO:tensorflow:loss = 3899.0068, step = 7202 (47.634 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05505\n",
      "INFO:tensorflow:loss = 3394.6057, step = 7302 (48.660 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06935\n",
      "INFO:tensorflow:loss = 3301.7925, step = 7402 (48.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09409\n",
      "INFO:tensorflow:loss = 3123.2566, step = 7502 (47.753 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7544 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2975.6729.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-7544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 302.52it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:46<00:00, 46.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.0631805669543969\n",
      "[test] Raw Hits@1: 4.84\n",
      "[test] Raw Hits@3: 6.54\n",
      "[test] Raw Hits@5: 7.359999999999999\n",
      "[test] Raw Hits@10: 8.799999999999999\n",
      "[test] Filtered MRR: 0.06618374486424694\n",
      "[test] Filtered Hits@1: 5.06\n",
      "[test] Filtered Hits@3: 6.859999999999999\n",
      "[test] Filtered Hits@5: 7.720000000000001\n",
      "[test] Filtered Hits@10: 9.32\n",
      "INFO:tensorflow:Epoch 9\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-7544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7545 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 2921.8972, step = 7545\n",
      "INFO:tensorflow:global_step/sec: 2.007\n",
      "INFO:tensorflow:loss = 3098.0432, step = 7645 (49.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01448\n",
      "INFO:tensorflow:loss = 3074.5027, step = 7745 (49.641 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03931\n",
      "INFO:tensorflow:loss = 2872.6753, step = 7845 (49.036 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98759\n",
      "INFO:tensorflow:loss = 2920.7244, step = 7945 (50.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99085\n",
      "INFO:tensorflow:loss = 2686.4211, step = 8045 (50.229 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98729\n",
      "INFO:tensorflow:loss = 2726.3306, step = 8145 (50.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99357\n",
      "INFO:tensorflow:loss = 2910.2344, step = 8245 (50.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02505\n",
      "INFO:tensorflow:loss = 2943.4736, step = 8345 (49.382 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90069\n",
      "INFO:tensorflow:loss = 2731.8574, step = 8445 (52.612 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 8487 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2598.4138.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-8487\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 286.27it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.09711139142673911\n",
      "[test] Raw Hits@1: 7.64\n",
      "[test] Raw Hits@3: 10.18\n",
      "[test] Raw Hits@5: 11.72\n",
      "[test] Raw Hits@10: 13.900000000000002\n",
      "[test] Filtered MRR: 0.10184948849485101\n",
      "[test] Filtered Hits@1: 8.06\n",
      "[test] Filtered Hits@3: 10.6\n",
      "[test] Filtered Hits@5: 12.32\n",
      "[test] Filtered Hits@10: 14.48\n",
      "INFO:tensorflow:Epoch 10\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-8487\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 8488 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 2524.1445, step = 8488\n",
      "INFO:tensorflow:global_step/sec: 1.83909\n",
      "INFO:tensorflow:loss = 2826.584, step = 8588 (54.376 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98424\n",
      "INFO:tensorflow:loss = 2454.06, step = 8688 (50.397 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88109\n",
      "INFO:tensorflow:loss = 2248.9797, step = 8788 (53.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.86941\n",
      "INFO:tensorflow:loss = 2600.6045, step = 8888 (53.493 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90482\n",
      "INFO:tensorflow:loss = 2379.437, step = 8988 (52.499 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92695\n",
      "INFO:tensorflow:loss = 2392.8545, step = 9088 (51.895 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90684\n",
      "INFO:tensorflow:loss = 2207.913, step = 9188 (52.442 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99504\n",
      "INFO:tensorflow:loss = 2280.4346, step = 9288 (50.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98019\n",
      "INFO:tensorflow:loss = 2108.5813, step = 9388 (50.500 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 9430 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2195.0847.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-9430\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 293.13it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.13691361904961086\n",
      "[test] Raw Hits@1: 11.12\n",
      "[test] Raw Hits@3: 14.7\n",
      "[test] Raw Hits@5: 16.1\n",
      "[test] Raw Hits@10: 18.240000000000002\n",
      "[test] Filtered MRR: 0.14347594248787915\n",
      "[test] Filtered Hits@1: 11.700000000000001\n",
      "[test] Filtered Hits@3: 15.42\n",
      "[test] Filtered Hits@5: 16.84\n",
      "[test] Filtered Hits@10: 19.08\n",
      "INFO:tensorflow:Epoch 11\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-9430\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 9431 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 2101.8108, step = 9431\n",
      "INFO:tensorflow:global_step/sec: 2.03076\n",
      "INFO:tensorflow:loss = 2332.0264, step = 9531 (49.244 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92781\n",
      "INFO:tensorflow:loss = 2044.7534, step = 9631 (51.873 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97971\n",
      "INFO:tensorflow:loss = 2093.0635, step = 9731 (50.512 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93676\n",
      "INFO:tensorflow:loss = 1920.6118, step = 9831 (51.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96056\n",
      "INFO:tensorflow:loss = 2027.1832, step = 9931 (51.006 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96805\n",
      "INFO:tensorflow:loss = 1750.7977, step = 10031 (50.812 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03792\n",
      "INFO:tensorflow:loss = 1899.1565, step = 10131 (49.070 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06802\n",
      "INFO:tensorflow:loss = 1690.0786, step = 10231 (48.355 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02826\n",
      "INFO:tensorflow:loss = 1903.4823, step = 10331 (49.303 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10373 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 1700.338.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-10373\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 288.79it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.1886116762088887\n",
      "[test] Raw Hits@1: 15.920000000000002\n",
      "[test] Raw Hits@3: 19.98\n",
      "[test] Raw Hits@5: 21.72\n",
      "[test] Raw Hits@10: 24.04\n",
      "[test] Filtered MRR: 0.20094620954938044\n",
      "[test] Filtered Hits@1: 17.34\n",
      "[test] Filtered Hits@3: 21.08\n",
      "[test] Filtered Hits@5: 22.720000000000002\n",
      "[test] Filtered Hits@10: 25.080000000000002\n",
      "INFO:tensorflow:Epoch 12\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-10373\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 10374 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 1610.7825, step = 10374\n",
      "INFO:tensorflow:global_step/sec: 2.01632\n",
      "INFO:tensorflow:loss = 1611.9636, step = 10474 (49.597 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95801\n",
      "INFO:tensorflow:loss = 1574.335, step = 10574 (51.072 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99098\n",
      "INFO:tensorflow:loss = 1574.1996, step = 10674 (50.227 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9552\n",
      "INFO:tensorflow:loss = 1510.6879, step = 10774 (51.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.84643\n",
      "INFO:tensorflow:loss = 1541.0111, step = 10874 (54.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89802\n",
      "INFO:tensorflow:loss = 1490.3794, step = 10974 (52.687 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95715\n",
      "INFO:tensorflow:loss = 1428.0657, step = 11074 (51.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94893\n",
      "INFO:tensorflow:loss = 1566.7958, step = 11174 (51.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00727\n",
      "INFO:tensorflow:loss = 1481.9963, step = 11274 (49.819 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 11316 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 1152.0326.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-11316\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.65it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:48<00:00, 45.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.23125504266253183\n",
      "[test] Raw Hits@1: 19.48\n",
      "[test] Raw Hits@3: 24.54\n",
      "[test] Raw Hits@5: 26.939999999999998\n",
      "[test] Raw Hits@10: 29.92\n",
      "[test] Filtered MRR: 0.2492405089307379\n",
      "[test] Filtered Hits@1: 21.6\n",
      "[test] Filtered Hits@3: 25.979999999999997\n",
      "[test] Filtered Hits@5: 28.32\n",
      "[test] Filtered Hits@10: 31.240000000000002\n",
      "INFO:tensorflow:Epoch 13\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-11316\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 11317 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 1475.3451, step = 11317\n",
      "INFO:tensorflow:global_step/sec: 1.909\n",
      "INFO:tensorflow:loss = 1244.5024, step = 11417 (52.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90012\n",
      "INFO:tensorflow:loss = 1323.0442, step = 11517 (52.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91958\n",
      "INFO:tensorflow:loss = 1377.4735, step = 11617 (52.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94068\n",
      "INFO:tensorflow:loss = 1481.5408, step = 11717 (51.528 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94009\n",
      "INFO:tensorflow:loss = 1221.3514, step = 11817 (51.544 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95014\n",
      "INFO:tensorflow:loss = 1259.9077, step = 11917 (51.279 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89198\n",
      "INFO:tensorflow:loss = 1274.3931, step = 12017 (52.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91492\n",
      "INFO:tensorflow:loss = 1072.0039, step = 12117 (52.222 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93514\n",
      "INFO:tensorflow:loss = 1106.3326, step = 12217 (51.676 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 12259 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 1026.673.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-12259\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 289.72it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.2644303851095363\n",
      "[test] Raw Hits@1: 21.58\n",
      "[test] Raw Hits@3: 29.18\n",
      "[test] Raw Hits@5: 32.26\n",
      "[test] Raw Hits@10: 35.199999999999996\n",
      "[test] Filtered MRR: 0.29162823123208037\n",
      "[test] Filtered Hits@1: 24.959999999999997\n",
      "[test] Filtered Hits@3: 31.480000000000004\n",
      "[test] Filtered Hits@5: 34.1\n",
      "[test] Filtered Hits@10: 36.58\n",
      "INFO:tensorflow:Epoch 14\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-12259\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 12260 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 995.62067, step = 12260\n",
      "INFO:tensorflow:global_step/sec: 1.86989\n",
      "INFO:tensorflow:loss = 948.54846, step = 12360 (53.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02293\n",
      "INFO:tensorflow:loss = 875.3078, step = 12460 (49.433 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99234\n",
      "INFO:tensorflow:loss = 1022.84, step = 12560 (50.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96086\n",
      "INFO:tensorflow:loss = 1026.9625, step = 12660 (50.998 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90434\n",
      "INFO:tensorflow:loss = 946.7599, step = 12760 (52.511 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91852\n",
      "INFO:tensorflow:loss = 1158.0503, step = 12860 (52.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89301\n",
      "INFO:tensorflow:loss = 916.413, step = 12960 (52.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98742\n",
      "INFO:tensorflow:loss = 1040.7043, step = 13060 (50.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05916\n",
      "INFO:tensorflow:loss = 881.1043, step = 13160 (48.564 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 13202 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 904.86163.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-13202\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 297.52it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.2851801322838064\n",
      "[test] Raw Hits@1: 22.759999999999998\n",
      "[test] Raw Hits@3: 31.72\n",
      "[test] Raw Hits@5: 35.260000000000005\n",
      "[test] Raw Hits@10: 38.7\n",
      "[test] Filtered MRR: 0.31808480449376114\n",
      "[test] Filtered Hits@1: 26.919999999999998\n",
      "[test] Filtered Hits@3: 34.4\n",
      "[test] Filtered Hits@5: 37.28\n",
      "[test] Filtered Hits@10: 40.2\n",
      "INFO:tensorflow:Epoch 15\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-13202\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 13203 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 786.9085, step = 13203\n",
      "INFO:tensorflow:global_step/sec: 1.87794\n",
      "INFO:tensorflow:loss = 862.1565, step = 13303 (53.251 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89699\n",
      "INFO:tensorflow:loss = 738.0839, step = 13403 (52.715 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.84181\n",
      "INFO:tensorflow:loss = 824.4739, step = 13503 (54.294 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95649\n",
      "INFO:tensorflow:loss = 659.2864, step = 13603 (51.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94354\n",
      "INFO:tensorflow:loss = 775.7995, step = 13703 (51.453 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.86472\n",
      "INFO:tensorflow:loss = 680.7427, step = 13803 (53.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93553\n",
      "INFO:tensorflow:loss = 698.95, step = 13903 (51.665 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.77391\n",
      "INFO:tensorflow:loss = 806.15234, step = 14003 (56.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90518\n",
      "INFO:tensorflow:loss = 646.8042, step = 14103 (52.488 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 14145 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 656.04565.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-14145\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 289.16it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:53<00:00, 44.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.30417740218419925\n",
      "[test] Raw Hits@1: 23.380000000000003\n",
      "[test] Raw Hits@3: 34.78\n",
      "[test] Raw Hits@5: 38.800000000000004\n",
      "[test] Raw Hits@10: 43.14\n",
      "[test] Filtered MRR: 0.346810258399864\n",
      "[test] Filtered Hits@1: 28.68\n",
      "[test] Filtered Hits@3: 38.22\n",
      "[test] Filtered Hits@5: 41.42\n",
      "[test] Filtered Hits@10: 45.1\n",
      "INFO:tensorflow:Epoch 16\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-14145\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 14146 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 566.7108, step = 14146\n",
      "INFO:tensorflow:global_step/sec: 1.92246\n",
      "INFO:tensorflow:loss = 655.64716, step = 14246 (52.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92917\n",
      "INFO:tensorflow:loss = 612.47974, step = 14346 (51.836 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88708\n",
      "INFO:tensorflow:loss = 646.8537, step = 14446 (52.992 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94855\n",
      "INFO:tensorflow:loss = 605.0496, step = 14546 (51.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88633\n",
      "INFO:tensorflow:loss = 677.4255, step = 14646 (53.013 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97258\n",
      "INFO:tensorflow:loss = 620.23737, step = 14746 (50.695 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97829\n",
      "INFO:tensorflow:loss = 668.1877, step = 14846 (50.548 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90976\n",
      "INFO:tensorflow:loss = 588.6731, step = 14946 (52.363 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03918\n",
      "INFO:tensorflow:loss = 575.7958, step = 15046 (49.039 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 15088 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 591.3831.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-15088\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 287.46it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.3158079372010017\n",
      "[test] Raw Hits@1: 23.68\n",
      "[test] Raw Hits@3: 36.24\n",
      "[test] Raw Hits@5: 41.28\n",
      "[test] Raw Hits@10: 45.82\n",
      "[test] Filtered MRR: 0.36523358628169206\n",
      "[test] Filtered Hits@1: 29.68\n",
      "[test] Filtered Hits@3: 40.660000000000004\n",
      "[test] Filtered Hits@5: 44.7\n",
      "[test] Filtered Hits@10: 48.339999999999996\n",
      "INFO:tensorflow:Epoch 17\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-15088\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 15089 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 511.14578, step = 15089\n",
      "INFO:tensorflow:global_step/sec: 1.9467\n",
      "INFO:tensorflow:loss = 500.7829, step = 15189 (51.371 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97336\n",
      "INFO:tensorflow:loss = 572.03937, step = 15289 (50.674 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03969\n",
      "INFO:tensorflow:loss = 454.27942, step = 15389 (49.027 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92978\n",
      "INFO:tensorflow:loss = 534.9379, step = 15489 (51.820 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02288\n",
      "INFO:tensorflow:loss = 428.47565, step = 15589 (49.434 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94093\n",
      "INFO:tensorflow:loss = 425.5141, step = 15689 (51.522 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87333\n",
      "INFO:tensorflow:loss = 548.2611, step = 15789 (53.381 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94498\n",
      "INFO:tensorflow:loss = 576.2822, step = 15889 (51.414 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92706\n",
      "INFO:tensorflow:loss = 514.7162, step = 15989 (51.893 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 16031 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 463.00928.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-16031\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 298.05it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.3238228219468193\n",
      "[test] Raw Hits@1: 23.48\n",
      "[test] Raw Hits@3: 37.76\n",
      "[test] Raw Hits@5: 43.1\n",
      "[test] Raw Hits@10: 48.4\n",
      "[test] Filtered MRR: 0.38118122292601037\n",
      "[test] Filtered Hits@1: 30.36\n",
      "[test] Filtered Hits@3: 43.04\n",
      "[test] Filtered Hits@5: 46.94\n",
      "[test] Filtered Hits@10: 51.5\n",
      "INFO:tensorflow:Epoch 18\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-16031\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 16032 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 443.22113, step = 16032\n",
      "INFO:tensorflow:global_step/sec: 1.95338\n",
      "INFO:tensorflow:loss = 354.83755, step = 16132 (51.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89296\n",
      "INFO:tensorflow:loss = 440.7857, step = 16232 (52.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93779\n",
      "INFO:tensorflow:loss = 415.85828, step = 16332 (51.605 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89605\n",
      "INFO:tensorflow:loss = 424.96637, step = 16432 (52.741 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.86232\n",
      "INFO:tensorflow:loss = 470.5042, step = 16532 (53.696 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88917\n",
      "INFO:tensorflow:loss = 446.7924, step = 16632 (52.934 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88666\n",
      "INFO:tensorflow:loss = 366.52377, step = 16732 (53.004 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.86068\n",
      "INFO:tensorflow:loss = 422.73547, step = 16832 (53.743 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91028\n",
      "INFO:tensorflow:loss = 453.23923, step = 16932 (52.349 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 16974 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 591.6472.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-16974\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 283.06it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.33727058539996857\n",
      "[test] Raw Hits@1: 24.18\n",
      "[test] Raw Hits@3: 39.26\n",
      "[test] Raw Hits@5: 44.98\n",
      "[test] Raw Hits@10: 51.06\n",
      "[test] Filtered MRR: 0.40119900372868617\n",
      "[test] Filtered Hits@1: 31.680000000000003\n",
      "[test] Filtered Hits@3: 45.14\n",
      "[test] Filtered Hits@5: 49.66\n",
      "[test] Filtered Hits@10: 54.74\n",
      "INFO:tensorflow:Epoch 19\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-16974\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 16975 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 349.23187, step = 16975\n",
      "INFO:tensorflow:global_step/sec: 1.91988\n",
      "INFO:tensorflow:loss = 383.36856, step = 17075 (52.088 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91174\n",
      "INFO:tensorflow:loss = 419.07254, step = 17175 (52.308 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90081\n",
      "INFO:tensorflow:loss = 438.43216, step = 17275 (52.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89023\n",
      "INFO:tensorflow:loss = 340.8766, step = 17375 (52.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06195\n",
      "INFO:tensorflow:loss = 328.87543, step = 17475 (48.497 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94083\n",
      "INFO:tensorflow:loss = 343.35867, step = 17575 (51.525 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85932\n",
      "INFO:tensorflow:loss = 371.13818, step = 17675 (53.783 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92983\n",
      "INFO:tensorflow:loss = 401.01294, step = 17775 (51.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90706\n",
      "INFO:tensorflow:loss = 410.40292, step = 17875 (52.437 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 17917 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 314.9132.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-17917\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 284.39it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.348837410215823\n",
      "[test] Raw Hits@1: 24.779999999999998\n",
      "[test] Raw Hits@3: 40.82\n",
      "[test] Raw Hits@5: 46.46\n",
      "[test] Raw Hits@10: 53.300000000000004\n",
      "[test] Filtered MRR: 0.41801103378910903\n",
      "[test] Filtered Hits@1: 32.78\n",
      "[test] Filtered Hits@3: 47.52\n",
      "[test] Filtered Hits@5: 51.9\n",
      "[test] Filtered Hits@10: 57.26\n",
      "INFO:tensorflow:Epoch 20\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-17917\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 17918 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 322.51492, step = 17918\n",
      "INFO:tensorflow:global_step/sec: 1.84851\n",
      "INFO:tensorflow:loss = 359.03098, step = 18018 (54.099 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98119\n",
      "INFO:tensorflow:loss = 346.96558, step = 18118 (50.475 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.08126\n",
      "INFO:tensorflow:loss = 348.772, step = 18218 (48.048 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98034\n",
      "INFO:tensorflow:loss = 321.3858, step = 18318 (50.496 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96318\n",
      "INFO:tensorflow:loss = 335.15945, step = 18418 (50.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95583\n",
      "INFO:tensorflow:loss = 303.57614, step = 18518 (51.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98727\n",
      "INFO:tensorflow:loss = 349.71292, step = 18618 (50.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90652\n",
      "INFO:tensorflow:loss = 319.91943, step = 18718 (52.452 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88937\n",
      "INFO:tensorflow:loss = 336.00555, step = 18818 (52.930 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 18860 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 298.15796.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-18860\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.62it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.3603743542940252\n",
      "[test] Raw Hits@1: 25.5\n",
      "[test] Raw Hits@3: 41.980000000000004\n",
      "[test] Raw Hits@5: 48.18\n",
      "[test] Raw Hits@10: 55.66\n",
      "[test] Filtered MRR: 0.432988459103845\n",
      "[test] Filtered Hits@1: 33.7\n",
      "[test] Filtered Hits@3: 49.04\n",
      "[test] Filtered Hits@5: 54.26\n",
      "[test] Filtered Hits@10: 59.86\n",
      "INFO:tensorflow:Epoch 21\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-18860\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 18861 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 312.29132, step = 18861\n",
      "INFO:tensorflow:global_step/sec: 1.91365\n",
      "INFO:tensorflow:loss = 331.4696, step = 18961 (52.257 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9315\n",
      "INFO:tensorflow:loss = 280.28503, step = 19061 (51.773 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90756\n",
      "INFO:tensorflow:loss = 314.06308, step = 19161 (52.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91789\n",
      "INFO:tensorflow:loss = 362.83948, step = 19261 (52.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87048\n",
      "INFO:tensorflow:loss = 281.8909, step = 19361 (53.462 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89669\n",
      "INFO:tensorflow:loss = 363.67712, step = 19461 (52.723 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9205\n",
      "INFO:tensorflow:loss = 265.11972, step = 19561 (52.069 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85332\n",
      "INFO:tensorflow:loss = 310.57925, step = 19661 (53.958 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88814\n",
      "INFO:tensorflow:loss = 300.37683, step = 19761 (52.962 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 19803 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 253.32352.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-19803\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 287.20it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.36817461251898786\n",
      "[test] Raw Hits@1: 25.679999999999996\n",
      "[test] Raw Hits@3: 43.08\n",
      "[test] Raw Hits@5: 49.64\n",
      "[test] Raw Hits@10: 57.199999999999996\n",
      "[test] Filtered MRR: 0.44947417883094315\n",
      "[test] Filtered Hits@1: 34.74\n",
      "[test] Filtered Hits@3: 51.25999999999999\n",
      "[test] Filtered Hits@5: 56.52\n",
      "[test] Filtered Hits@10: 62.72\n",
      "INFO:tensorflow:Epoch 22\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-19803\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 19804 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 215.58069, step = 19804\n",
      "INFO:tensorflow:global_step/sec: 1.83648\n",
      "INFO:tensorflow:loss = 245.18816, step = 19904 (54.453 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92586\n",
      "INFO:tensorflow:loss = 214.47845, step = 20004 (51.925 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89713\n",
      "INFO:tensorflow:loss = 259.57144, step = 20104 (52.711 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.80857\n",
      "INFO:tensorflow:loss = 304.17026, step = 20204 (55.292 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93922\n",
      "INFO:tensorflow:loss = 296.967, step = 20304 (51.567 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85062\n",
      "INFO:tensorflow:loss = 238.95981, step = 20404 (54.036 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93279\n",
      "INFO:tensorflow:loss = 316.60388, step = 20504 (51.739 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93079\n",
      "INFO:tensorflow:loss = 245.18805, step = 20604 (51.792 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93611\n",
      "INFO:tensorflow:loss = 217.10481, step = 20704 (51.650 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20746 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 241.11078.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-20746\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.16it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.3773616519676515\n",
      "[test] Raw Hits@1: 26.16\n",
      "[test] Raw Hits@3: 43.84\n",
      "[test] Raw Hits@5: 51.64\n",
      "[test] Raw Hits@10: 60.199999999999996\n",
      "[test] Filtered MRR: 0.4690233095071274\n",
      "[test] Filtered Hits@1: 36.46\n",
      "[test] Filtered Hits@3: 52.900000000000006\n",
      "[test] Filtered Hits@5: 59.5\n",
      "[test] Filtered Hits@10: 65.88000000000001\n",
      "INFO:tensorflow:Epoch 23\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-20746\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 20747 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 225.6483, step = 20747\n",
      "INFO:tensorflow:global_step/sec: 1.84779\n",
      "INFO:tensorflow:loss = 263.47095, step = 20847 (54.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.77162\n",
      "INFO:tensorflow:loss = 226.09676, step = 20947 (56.445 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00953\n",
      "INFO:tensorflow:loss = 176.62537, step = 21047 (49.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90767\n",
      "INFO:tensorflow:loss = 279.81354, step = 21147 (52.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82504\n",
      "INFO:tensorflow:loss = 215.99225, step = 21247 (54.793 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.8509\n",
      "INFO:tensorflow:loss = 292.5693, step = 21347 (54.028 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91781\n",
      "INFO:tensorflow:loss = 259.85056, step = 21447 (52.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88908\n",
      "INFO:tensorflow:loss = 297.20126, step = 21547 (52.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85248\n",
      "INFO:tensorflow:loss = 310.28336, step = 21647 (53.982 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21689 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 232.48889.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-21689\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 296.87it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.387387847273987\n",
      "[test] Raw Hits@1: 27.1\n",
      "[test] Raw Hits@3: 44.56\n",
      "[test] Raw Hits@5: 52.459999999999994\n",
      "[test] Raw Hits@10: 61.480000000000004\n",
      "[test] Filtered MRR: 0.4806058926564042\n",
      "[test] Filtered Hits@1: 37.34\n",
      "[test] Filtered Hits@3: 54.76\n",
      "[test] Filtered Hits@5: 60.5\n",
      "[test] Filtered Hits@10: 67.22\n",
      "INFO:tensorflow:Epoch 24\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-21689\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 21690 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 247.88556, step = 21690\n",
      "INFO:tensorflow:global_step/sec: 1.96964\n",
      "INFO:tensorflow:loss = 242.3993, step = 21790 (50.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93919\n",
      "INFO:tensorflow:loss = 214.15692, step = 21890 (51.568 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82301\n",
      "INFO:tensorflow:loss = 198.304, step = 21990 (54.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00484\n",
      "INFO:tensorflow:loss = 215.54868, step = 22090 (49.879 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93901\n",
      "INFO:tensorflow:loss = 250.87335, step = 22190 (51.573 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90117\n",
      "INFO:tensorflow:loss = 257.77835, step = 22290 (52.599 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.8391\n",
      "INFO:tensorflow:loss = 212.0788, step = 22390 (54.374 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92168\n",
      "INFO:tensorflow:loss = 189.08026, step = 22490 (52.038 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88657\n",
      "INFO:tensorflow:loss = 223.84174, step = 22590 (53.006 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 22632 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 232.40752.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-22632\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 287.80it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:52<00:00, 44.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.3925281264554561\n",
      "[test] Raw Hits@1: 27.22\n",
      "[test] Raw Hits@3: 46.22\n",
      "[test] Raw Hits@5: 53.480000000000004\n",
      "[test] Raw Hits@10: 62.480000000000004\n",
      "[test] Filtered MRR: 0.4869617086421429\n",
      "[test] Filtered Hits@1: 37.7\n",
      "[test] Filtered Hits@3: 55.92\n",
      "[test] Filtered Hits@5: 62.019999999999996\n",
      "[test] Filtered Hits@10: 68.10000000000001\n",
      "INFO:tensorflow:Epoch 25\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-22632\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 22633 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 179.5658, step = 22633\n",
      "INFO:tensorflow:global_step/sec: 1.92094\n",
      "INFO:tensorflow:loss = 193.2716, step = 22733 (52.059 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19816\n",
      "INFO:tensorflow:loss = 216.505, step = 22833 (45.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19393\n",
      "INFO:tensorflow:loss = 160.36896, step = 22933 (45.580 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.10096\n",
      "INFO:tensorflow:loss = 192.08511, step = 23033 (47.597 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18869\n",
      "INFO:tensorflow:loss = 170.30385, step = 23133 (45.689 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.13539\n",
      "INFO:tensorflow:loss = 292.61902, step = 23233 (46.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19016\n",
      "INFO:tensorflow:loss = 192.62404, step = 23333 (45.658 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19654\n",
      "INFO:tensorflow:loss = 189.81119, step = 23433 (45.527 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19609\n",
      "INFO:tensorflow:loss = 207.42284, step = 23533 (45.535 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 23575 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 181.54623.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-23575\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.81it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.40051480870709105\n",
      "[test] Raw Hits@1: 27.88\n",
      "[test] Raw Hits@3: 46.98\n",
      "[test] Raw Hits@5: 54.620000000000005\n",
      "[test] Raw Hits@10: 63.160000000000004\n",
      "[test] Filtered MRR: 0.5011835763591868\n",
      "[test] Filtered Hits@1: 39.6\n",
      "[test] Filtered Hits@3: 56.699999999999996\n",
      "[test] Filtered Hits@5: 62.5\n",
      "[test] Filtered Hits@10: 69.17999999999999\n",
      "INFO:tensorflow:Epoch 26\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-23575\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 23576 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 182.29103, step = 23576\n",
      "INFO:tensorflow:global_step/sec: 2.08104\n",
      "INFO:tensorflow:loss = 206.23619, step = 23676 (48.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19396\n",
      "INFO:tensorflow:loss = 158.97221, step = 23776 (45.580 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19407\n",
      "INFO:tensorflow:loss = 250.18353, step = 23876 (45.578 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19689\n",
      "INFO:tensorflow:loss = 150.47366, step = 23976 (45.518 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1924\n",
      "INFO:tensorflow:loss = 145.62955, step = 24076 (45.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19588\n",
      "INFO:tensorflow:loss = 201.50371, step = 24176 (45.540 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19347\n",
      "INFO:tensorflow:loss = 200.48053, step = 24276 (45.590 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20064\n",
      "INFO:tensorflow:loss = 232.07495, step = 24376 (45.441 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19252\n",
      "INFO:tensorflow:loss = 199.42085, step = 24476 (45.610 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 24518 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 161.4162.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-24518\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.69it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4005241468426218\n",
      "[test] Raw Hits@1: 27.3\n",
      "[test] Raw Hits@3: 47.52\n",
      "[test] Raw Hits@5: 55.1\n",
      "[test] Raw Hits@10: 64.05999999999999\n",
      "[test] Filtered MRR: 0.5043928134119973\n",
      "[test] Filtered Hits@1: 38.98\n",
      "[test] Filtered Hits@3: 58.18\n",
      "[test] Filtered Hits@5: 64.05999999999999\n",
      "[test] Filtered Hits@10: 70.42\n",
      "INFO:tensorflow:Epoch 27\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-24518\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 24519 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 161.3692, step = 24519\n",
      "INFO:tensorflow:global_step/sec: 2.18768\n",
      "INFO:tensorflow:loss = 197.99605, step = 24619 (45.712 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19611\n",
      "INFO:tensorflow:loss = 202.16142, step = 24719 (45.535 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19428\n",
      "INFO:tensorflow:loss = 161.96579, step = 24819 (45.573 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19752\n",
      "INFO:tensorflow:loss = 188.5465, step = 24919 (45.506 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20421\n",
      "INFO:tensorflow:loss = 238.41096, step = 25019 (45.368 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19777\n",
      "INFO:tensorflow:loss = 187.41187, step = 25119 (45.501 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19957\n",
      "INFO:tensorflow:loss = 181.03702, step = 25219 (45.463 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20294\n",
      "INFO:tensorflow:loss = 197.74037, step = 25319 (45.394 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1992\n",
      "INFO:tensorflow:loss = 192.57605, step = 25419 (45.471 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 25461 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 216.63821.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-25461\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.46it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:41<00:00, 49.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4079258279712553\n",
      "[test] Raw Hits@1: 28.52\n",
      "[test] Raw Hits@3: 47.82\n",
      "[test] Raw Hits@5: 55.82\n",
      "[test] Raw Hits@10: 64.24\n",
      "[test] Filtered MRR: 0.5135488616605876\n",
      "[test] Filtered Hits@1: 40.46\n",
      "[test] Filtered Hits@3: 58.42\n",
      "[test] Filtered Hits@5: 64.0\n",
      "[test] Filtered Hits@10: 70.62\n",
      "INFO:tensorflow:Epoch 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-25461\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 25462 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 154.07501, step = 25462\n",
      "INFO:tensorflow:global_step/sec: 1.86799\n",
      "INFO:tensorflow:loss = 162.29115, step = 25562 (53.535 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.73329\n",
      "INFO:tensorflow:loss = 186.94153, step = 25662 (57.694 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87133\n",
      "INFO:tensorflow:loss = 147.81194, step = 25762 (53.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95273\n",
      "INFO:tensorflow:loss = 148.86191, step = 25862 (51.210 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.77366\n",
      "INFO:tensorflow:loss = 157.68977, step = 25962 (56.380 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.64764\n",
      "INFO:tensorflow:loss = 160.94623, step = 26062 (60.693 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.64059\n",
      "INFO:tensorflow:loss = 178.2045, step = 26162 (60.954 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.6205\n",
      "INFO:tensorflow:loss = 235.39514, step = 26262 (61.709 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.70335\n",
      "INFO:tensorflow:loss = 168.15633, step = 26362 (58.708 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 26404 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 245.77858.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-26404\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:18<00:00, 265.86it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4129387060334115\n",
      "[test] Raw Hits@1: 28.58\n",
      "[test] Raw Hits@3: 48.82\n",
      "[test] Raw Hits@5: 56.02\n",
      "[test] Raw Hits@10: 64.78\n",
      "[test] Filtered MRR: 0.521065600977317\n",
      "[test] Filtered Hits@1: 41.0\n",
      "[test] Filtered Hits@3: 59.160000000000004\n",
      "[test] Filtered Hits@5: 65.44\n",
      "[test] Filtered Hits@10: 71.78\n",
      "INFO:tensorflow:Epoch 29\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-26404\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 26405 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 163.9878, step = 26405\n",
      "INFO:tensorflow:global_step/sec: 1.74721\n",
      "INFO:tensorflow:loss = 158.87045, step = 26505 (57.236 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03918\n",
      "INFO:tensorflow:loss = 127.776436, step = 26605 (49.039 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03469\n",
      "INFO:tensorflow:loss = 128.01459, step = 26705 (49.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94339\n",
      "INFO:tensorflow:loss = 156.71764, step = 26805 (51.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96936\n",
      "INFO:tensorflow:loss = 202.46672, step = 26905 (50.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95206\n",
      "INFO:tensorflow:loss = 162.20123, step = 27005 (51.228 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99732\n",
      "INFO:tensorflow:loss = 138.80342, step = 27105 (50.067 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01449\n",
      "INFO:tensorflow:loss = 155.82281, step = 27205 (49.640 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93661\n",
      "INFO:tensorflow:loss = 206.67332, step = 27305 (51.638 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 27347 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 138.87065.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-27347\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 283.14it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.42040341258661446\n",
      "[test] Raw Hits@1: 29.32\n",
      "[test] Raw Hits@3: 49.32\n",
      "[test] Raw Hits@5: 57.14\n",
      "[test] Raw Hits@10: 66.14\n",
      "[test] Filtered MRR: 0.5367750029998566\n",
      "[test] Filtered Hits@1: 42.38\n",
      "[test] Filtered Hits@3: 61.3\n",
      "[test] Filtered Hits@5: 66.72\n",
      "[test] Filtered Hits@10: 73.08\n",
      "INFO:tensorflow:Epoch 30\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-27347\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 27348 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 148.57367, step = 27348\n",
      "INFO:tensorflow:global_step/sec: 1.87149\n",
      "INFO:tensorflow:loss = 138.57399, step = 27448 (53.435 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91984\n",
      "INFO:tensorflow:loss = 136.39906, step = 27548 (52.087 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.63563\n",
      "INFO:tensorflow:loss = 121.067215, step = 27648 (61.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91398\n",
      "INFO:tensorflow:loss = 161.19292, step = 27748 (52.247 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.80761\n",
      "INFO:tensorflow:loss = 180.78918, step = 27848 (55.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.78788\n",
      "INFO:tensorflow:loss = 141.50383, step = 27948 (55.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.84445\n",
      "INFO:tensorflow:loss = 157.66641, step = 28048 (54.215 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.79871\n",
      "INFO:tensorflow:loss = 172.15825, step = 28148 (55.596 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.6078\n",
      "INFO:tensorflow:loss = 144.53154, step = 28248 (62.197 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 28290 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 188.28651.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-28290\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:21<00:00, 235.68it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.42029523816301506\n",
      "[test] Raw Hits@1: 29.360000000000003\n",
      "[test] Raw Hits@3: 49.8\n",
      "[test] Raw Hits@5: 57.42\n",
      "[test] Raw Hits@10: 65.78\n",
      "[test] Filtered MRR: 0.5396123823811042\n",
      "[test] Filtered Hits@1: 43.3\n",
      "[test] Filtered Hits@3: 61.199999999999996\n",
      "[test] Filtered Hits@5: 66.25999999999999\n",
      "[test] Filtered Hits@10: 72.82\n",
      "INFO:tensorflow:Epoch 31\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-28290\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 28291 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 125.632576, step = 28291\n",
      "INFO:tensorflow:global_step/sec: 1.86071\n",
      "INFO:tensorflow:loss = 125.85505, step = 28391 (53.744 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01075\n",
      "INFO:tensorflow:loss = 160.76031, step = 28491 (49.732 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94319\n",
      "INFO:tensorflow:loss = 143.55011, step = 28591 (51.462 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.7891\n",
      "INFO:tensorflow:loss = 137.70094, step = 28691 (55.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85822\n",
      "INFO:tensorflow:loss = 140.17003, step = 28791 (53.815 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91939\n",
      "INFO:tensorflow:loss = 130.74927, step = 28891 (52.100 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.76148\n",
      "INFO:tensorflow:loss = 102.34077, step = 28991 (56.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85934\n",
      "INFO:tensorflow:loss = 166.29271, step = 29091 (53.783 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0671\n",
      "INFO:tensorflow:loss = 159.0891, step = 29191 (48.378 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 29233 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 125.24172.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-29233\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 298.67it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:48<00:00, 46.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4230974113636963\n",
      "[test] Raw Hits@1: 29.360000000000003\n",
      "[test] Raw Hits@3: 50.080000000000005\n",
      "[test] Raw Hits@5: 58.28\n",
      "[test] Raw Hits@10: 66.92\n",
      "[test] Filtered MRR: 0.5449976615811863\n",
      "[test] Filtered Hits@1: 43.3\n",
      "[test] Filtered Hits@3: 62.22\n",
      "[test] Filtered Hits@5: 68.0\n",
      "[test] Filtered Hits@10: 74.08\n",
      "INFO:tensorflow:Epoch 32\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-29233\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 29234 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 97.66217, step = 29234\n",
      "INFO:tensorflow:global_step/sec: 1.81507\n",
      "INFO:tensorflow:loss = 181.18413, step = 29334 (55.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97597\n",
      "INFO:tensorflow:loss = 145.60747, step = 29434 (50.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87\n",
      "INFO:tensorflow:loss = 127.58386, step = 29534 (53.476 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.63131\n",
      "INFO:tensorflow:loss = 129.60197, step = 29634 (61.300 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88376\n",
      "INFO:tensorflow:loss = 185.04787, step = 29734 (53.087 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82362\n",
      "INFO:tensorflow:loss = 141.06778, step = 29834 (54.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.83198\n",
      "INFO:tensorflow:loss = 149.25858, step = 29934 (54.586 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.80887\n",
      "INFO:tensorflow:loss = 155.11937, step = 30034 (55.284 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93409\n",
      "INFO:tensorflow:loss = 205.37596, step = 30134 (51.703 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 30176 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 150.26582.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-30176\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 284.46it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4272099415574219\n",
      "[test] Raw Hits@1: 29.94\n",
      "[test] Raw Hits@3: 50.12\n",
      "[test] Raw Hits@5: 58.42\n",
      "[test] Raw Hits@10: 67.2\n",
      "[test] Filtered MRR: 0.5482905495194951\n",
      "[test] Filtered Hits@1: 43.64\n",
      "[test] Filtered Hits@3: 62.18\n",
      "[test] Filtered Hits@5: 68.02\n",
      "[test] Filtered Hits@10: 74.2\n",
      "INFO:tensorflow:Epoch 33\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-30176\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 30177 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 105.5841, step = 30177\n",
      "INFO:tensorflow:global_step/sec: 1.90778\n",
      "INFO:tensorflow:loss = 167.2845, step = 30277 (52.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94797\n",
      "INFO:tensorflow:loss = 109.916084, step = 30377 (51.335 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09077\n",
      "INFO:tensorflow:loss = 117.570816, step = 30477 (47.829 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98998\n",
      "INFO:tensorflow:loss = 130.56325, step = 30577 (50.252 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97282\n",
      "INFO:tensorflow:loss = 167.35898, step = 30677 (50.689 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95359\n",
      "INFO:tensorflow:loss = 141.95818, step = 30777 (51.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.022\n",
      "INFO:tensorflow:loss = 120.61192, step = 30877 (49.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09107\n",
      "INFO:tensorflow:loss = 142.71913, step = 30977 (47.822 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18363\n",
      "INFO:tensorflow:loss = 166.52309, step = 31077 (45.796 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 31119 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 141.28532.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-31119\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 299.81it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:43<00:00, 48.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4284318763623325\n",
      "[test] Raw Hits@1: 30.0\n",
      "[test] Raw Hits@3: 50.54\n",
      "[test] Raw Hits@5: 58.379999999999995\n",
      "[test] Raw Hits@10: 67.08\n",
      "[test] Filtered MRR: 0.5490962829950463\n",
      "[test] Filtered Hits@1: 44.080000000000005\n",
      "[test] Filtered Hits@3: 61.88\n",
      "[test] Filtered Hits@5: 67.84\n",
      "[test] Filtered Hits@10: 74.32\n",
      "INFO:tensorflow:Epoch 34\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-31119\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 31120 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 102.38419, step = 31120\n",
      "INFO:tensorflow:global_step/sec: 2.17469\n",
      "INFO:tensorflow:loss = 148.47293, step = 31220 (45.985 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17518\n",
      "INFO:tensorflow:loss = 133.41821, step = 31320 (45.973 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18565\n",
      "INFO:tensorflow:loss = 152.84636, step = 31420 (45.753 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.08574\n",
      "INFO:tensorflow:loss = 177.04308, step = 31520 (47.945 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.73446\n",
      "INFO:tensorflow:loss = 106.38487, step = 31620 (57.658 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.6484\n",
      "INFO:tensorflow:loss = 103.63095, step = 31720 (60.662 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.58744\n",
      "INFO:tensorflow:loss = 163.36028, step = 31820 (62.994 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.59597\n",
      "INFO:tensorflow:loss = 135.08792, step = 31920 (62.658 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.65048\n",
      "INFO:tensorflow:loss = 119.23503, step = 32020 (60.589 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 32062 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 107.536804.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-32062\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 283.92it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:47<00:00, 46.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4359879471373669\n",
      "[test] Raw Hits@1: 31.019999999999996\n",
      "[test] Raw Hits@3: 50.580000000000005\n",
      "[test] Raw Hits@5: 58.74\n",
      "[test] Raw Hits@10: 67.14\n",
      "[test] Filtered MRR: 0.5561986988246832\n",
      "[test] Filtered Hits@1: 44.68\n",
      "[test] Filtered Hits@3: 62.580000000000005\n",
      "[test] Filtered Hits@5: 68.38\n",
      "[test] Filtered Hits@10: 75.03999999999999\n",
      "INFO:tensorflow:Epoch 35\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-32062\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 32063 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 210.93018, step = 32063\n",
      "INFO:tensorflow:global_step/sec: 1.60687\n",
      "INFO:tensorflow:loss = 118.78428, step = 32163 (62.234 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.59006\n",
      "INFO:tensorflow:loss = 101.261536, step = 32263 (62.891 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.58607\n",
      "INFO:tensorflow:loss = 151.116, step = 32363 (63.049 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.60666\n",
      "INFO:tensorflow:loss = 143.19534, step = 32463 (62.242 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.58934\n",
      "INFO:tensorflow:loss = 94.113594, step = 32563 (62.918 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.56869\n",
      "INFO:tensorflow:loss = 132.51448, step = 32663 (63.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.58478\n",
      "INFO:tensorflow:loss = 139.90013, step = 32763 (63.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.59403\n",
      "INFO:tensorflow:loss = 105.20338, step = 32863 (62.733 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.58998\n",
      "INFO:tensorflow:loss = 150.4544, step = 32963 (62.894 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 33005 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 133.77145.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-33005\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:21<00:00, 235.72it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:47<00:00, 46.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4384471824353052\n",
      "[test] Raw Hits@1: 31.22\n",
      "[test] Raw Hits@3: 51.25999999999999\n",
      "[test] Raw Hits@5: 59.099999999999994\n",
      "[test] Raw Hits@10: 67.5\n",
      "[test] Filtered MRR: 0.5653524060028947\n",
      "[test] Filtered Hits@1: 45.92\n",
      "[test] Filtered Hits@3: 63.38\n",
      "[test] Filtered Hits@5: 69.04\n",
      "[test] Filtered Hits@10: 75.24\n",
      "INFO:tensorflow:Epoch 36\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-33005\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 33006 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 82.81326, step = 33006\n",
      "INFO:tensorflow:global_step/sec: 1.98609\n",
      "INFO:tensorflow:loss = 135.7933, step = 33106 (50.352 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07937\n",
      "INFO:tensorflow:loss = 136.28795, step = 33206 (48.091 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00473\n",
      "INFO:tensorflow:loss = 158.55016, step = 33306 (49.882 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.13455\n",
      "INFO:tensorflow:loss = 114.12029, step = 33406 (46.848 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.956\n",
      "INFO:tensorflow:loss = 145.83209, step = 33506 (51.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94369\n",
      "INFO:tensorflow:loss = 105.57735, step = 33606 (51.448 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97349\n",
      "INFO:tensorflow:loss = 130.8971, step = 33706 (50.672 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87136\n",
      "INFO:tensorflow:loss = 120.41518, step = 33806 (53.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87914\n",
      "INFO:tensorflow:loss = 104.644745, step = 33906 (53.216 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 33948 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 137.88812.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-33948\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 298.29it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4369405794552612\n",
      "[test] Raw Hits@1: 30.44\n",
      "[test] Raw Hits@3: 51.38\n",
      "[test] Raw Hits@5: 59.72\n",
      "[test] Raw Hits@10: 68.26\n",
      "[test] Filtered MRR: 0.5686790203710304\n",
      "[test] Filtered Hits@1: 45.46\n",
      "[test] Filtered Hits@3: 64.56\n",
      "[test] Filtered Hits@5: 70.74000000000001\n",
      "[test] Filtered Hits@10: 76.3\n",
      "INFO:tensorflow:Epoch 37\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-33948\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 33949 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 135.64905, step = 33949\n",
      "INFO:tensorflow:global_step/sec: 1.96883\n",
      "INFO:tensorflow:loss = 108.51066, step = 34049 (50.793 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00374\n",
      "INFO:tensorflow:loss = 114.327774, step = 34149 (49.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.14014\n",
      "INFO:tensorflow:loss = 107.304634, step = 34249 (46.726 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.13734\n",
      "INFO:tensorflow:loss = 98.25109, step = 34349 (46.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92681\n",
      "INFO:tensorflow:loss = 120.583496, step = 34449 (51.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.84786\n",
      "INFO:tensorflow:loss = 107.593346, step = 34549 (54.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9272\n",
      "INFO:tensorflow:loss = 87.84691, step = 34649 (51.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.86293\n",
      "INFO:tensorflow:loss = 125.74872, step = 34749 (53.679 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99817\n",
      "INFO:tensorflow:loss = 153.07767, step = 34849 (50.048 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 34891 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 131.3934.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-34891\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 301.10it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.44328035414973493\n",
      "[test] Raw Hits@1: 31.580000000000002\n",
      "[test] Raw Hits@3: 51.839999999999996\n",
      "[test] Raw Hits@5: 59.38\n",
      "[test] Raw Hits@10: 68.4\n",
      "[test] Filtered MRR: 0.5713099551075275\n",
      "[test] Filtered Hits@1: 46.22\n",
      "[test] Filtered Hits@3: 64.75999999999999\n",
      "[test] Filtered Hits@5: 69.89999999999999\n",
      "[test] Filtered Hits@10: 75.7\n",
      "INFO:tensorflow:Epoch 38\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-34891\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 34892 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 126.53105, step = 34892\n",
      "INFO:tensorflow:global_step/sec: 1.8626\n",
      "INFO:tensorflow:loss = 119.34332, step = 34992 (53.689 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.81053\n",
      "INFO:tensorflow:loss = 70.20335, step = 35092 (55.232 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96559\n",
      "INFO:tensorflow:loss = 121.60405, step = 35192 (50.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87834\n",
      "INFO:tensorflow:loss = 118.71575, step = 35292 (53.238 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9366\n",
      "INFO:tensorflow:loss = 75.825874, step = 35392 (51.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96686\n",
      "INFO:tensorflow:loss = 106.41298, step = 35492 (50.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02918\n",
      "INFO:tensorflow:loss = 134.8586, step = 35592 (49.281 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00627\n",
      "INFO:tensorflow:loss = 88.75265, step = 35692 (49.844 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98525\n",
      "INFO:tensorflow:loss = 95.74074, step = 35792 (50.372 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 35834 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 121.13367.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-35834\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 297.16it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4441404023373678\n",
      "[test] Raw Hits@1: 31.580000000000002\n",
      "[test] Raw Hits@3: 51.9\n",
      "[test] Raw Hits@5: 60.040000000000006\n",
      "[test] Raw Hits@10: 68.92\n",
      "[test] Filtered MRR: 0.5730535431973734\n",
      "[test] Filtered Hits@1: 46.339999999999996\n",
      "[test] Filtered Hits@3: 64.7\n",
      "[test] Filtered Hits@5: 70.8\n",
      "[test] Filtered Hits@10: 76.68\n",
      "INFO:tensorflow:Epoch 39\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-35834\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 35835 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 68.64652, step = 35835\n",
      "INFO:tensorflow:global_step/sec: 1.82643\n",
      "INFO:tensorflow:loss = 106.45157, step = 35935 (54.753 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.8859\n",
      "INFO:tensorflow:loss = 111.16191, step = 36035 (53.025 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91783\n",
      "INFO:tensorflow:loss = 124.06846, step = 36135 (52.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99355\n",
      "INFO:tensorflow:loss = 122.617035, step = 36235 (50.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03764\n",
      "INFO:tensorflow:loss = 114.68426, step = 36335 (49.077 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92457\n",
      "INFO:tensorflow:loss = 131.5405, step = 36435 (51.960 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03809\n",
      "INFO:tensorflow:loss = 135.57777, step = 36535 (49.065 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07003\n",
      "INFO:tensorflow:loss = 116.215515, step = 36635 (48.309 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97881\n",
      "INFO:tensorflow:loss = 122.08959, step = 36735 (50.536 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 36777 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 121.62319.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-36777\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 294.85it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.44570293864616467\n",
      "[test] Raw Hits@1: 31.56\n",
      "[test] Raw Hits@3: 52.080000000000005\n",
      "[test] Raw Hits@5: 60.480000000000004\n",
      "[test] Raw Hits@10: 69.44\n",
      "[test] Filtered MRR: 0.5747154763175241\n",
      "[test] Filtered Hits@1: 46.300000000000004\n",
      "[test] Filtered Hits@3: 64.84\n",
      "[test] Filtered Hits@5: 71.14\n",
      "[test] Filtered Hits@10: 77.22\n",
      "INFO:tensorflow:Epoch 40\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-36777\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 36778 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 86.24336, step = 36778\n",
      "INFO:tensorflow:global_step/sec: 1.93775\n",
      "INFO:tensorflow:loss = 72.188, step = 36878 (51.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90939\n",
      "INFO:tensorflow:loss = 83.66005, step = 36978 (52.372 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91064\n",
      "INFO:tensorflow:loss = 95.301994, step = 37078 (52.339 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9409\n",
      "INFO:tensorflow:loss = 79.45578, step = 37178 (51.522 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91056\n",
      "INFO:tensorflow:loss = 85.83821, step = 37278 (52.341 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95015\n",
      "INFO:tensorflow:loss = 103.13747, step = 37378 (51.278 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91455\n",
      "INFO:tensorflow:loss = 89.60957, step = 37478 (52.231 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92095\n",
      "INFO:tensorflow:loss = 100.79265, step = 37578 (52.058 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98001\n",
      "INFO:tensorflow:loss = 111.2351, step = 37678 (50.505 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 37720 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 118.28065.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-37720\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 298.95it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4496446058202497\n",
      "[test] Raw Hits@1: 32.16\n",
      "[test] Raw Hits@3: 52.459999999999994\n",
      "[test] Raw Hits@5: 60.660000000000004\n",
      "[test] Raw Hits@10: 69.3\n",
      "[test] Filtered MRR: 0.5843473639443926\n",
      "[test] Filtered Hits@1: 47.68\n",
      "[test] Filtered Hits@3: 66.0\n",
      "[test] Filtered Hits@5: 71.26\n",
      "[test] Filtered Hits@10: 77.62\n",
      "INFO:tensorflow:Epoch 41\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-37720\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 37721 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 104.70739, step = 37721\n",
      "INFO:tensorflow:global_step/sec: 1.89332\n",
      "INFO:tensorflow:loss = 88.415955, step = 37821 (52.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95684\n",
      "INFO:tensorflow:loss = 110.84213, step = 37921 (51.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93655\n",
      "INFO:tensorflow:loss = 105.77981, step = 38021 (51.638 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06045\n",
      "INFO:tensorflow:loss = 126.22665, step = 38121 (48.533 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93594\n",
      "INFO:tensorflow:loss = 106.55157, step = 38221 (51.655 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.8994\n",
      "INFO:tensorflow:loss = 143.10095, step = 38321 (52.648 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97323\n",
      "INFO:tensorflow:loss = 112.373116, step = 38421 (50.678 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05021\n",
      "INFO:tensorflow:loss = 112.56535, step = 38521 (48.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95735\n",
      "INFO:tensorflow:loss = 77.481415, step = 38621 (51.089 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 38663 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 101.38446.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-38663\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 291.67it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4495608500801407\n",
      "[test] Raw Hits@1: 32.04\n",
      "[test] Raw Hits@3: 52.94\n",
      "[test] Raw Hits@5: 60.72\n",
      "[test] Raw Hits@10: 69.1\n",
      "[test] Filtered MRR: 0.5843659181489794\n",
      "[test] Filtered Hits@1: 47.760000000000005\n",
      "[test] Filtered Hits@3: 65.53999999999999\n",
      "[test] Filtered Hits@5: 71.24000000000001\n",
      "[test] Filtered Hits@10: 77.0\n",
      "INFO:tensorflow:Epoch 42\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-38663\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 38664 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 106.66976, step = 38664\n",
      "INFO:tensorflow:global_step/sec: 1.9596\n",
      "INFO:tensorflow:loss = 106.576126, step = 38764 (51.032 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95826\n",
      "INFO:tensorflow:loss = 77.995056, step = 38864 (51.066 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90383\n",
      "INFO:tensorflow:loss = 106.63326, step = 38964 (52.526 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9277\n",
      "INFO:tensorflow:loss = 161.3577, step = 39064 (51.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92776\n",
      "INFO:tensorflow:loss = 105.442955, step = 39164 (51.874 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87701\n",
      "INFO:tensorflow:loss = 143.0438, step = 39264 (53.276 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98777\n",
      "INFO:tensorflow:loss = 128.08719, step = 39364 (50.308 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0047\n",
      "INFO:tensorflow:loss = 93.026054, step = 39464 (49.884 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92605\n",
      "INFO:tensorflow:loss = 77.214035, step = 39564 (51.919 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 39606 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 103.14431.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-39606\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 295.09it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.45570523693315035\n",
      "[test] Raw Hits@1: 32.800000000000004\n",
      "[test] Raw Hits@3: 53.26\n",
      "[test] Raw Hits@5: 60.760000000000005\n",
      "[test] Raw Hits@10: 69.76\n",
      "[test] Filtered MRR: 0.5938162776742277\n",
      "[test] Filtered Hits@1: 49.120000000000005\n",
      "[test] Filtered Hits@3: 65.96\n",
      "[test] Filtered Hits@5: 71.66\n",
      "[test] Filtered Hits@10: 77.52\n",
      "INFO:tensorflow:Epoch 43\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-39606\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 39607 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 119.88433, step = 39607\n",
      "INFO:tensorflow:global_step/sec: 1.91115\n",
      "INFO:tensorflow:loss = 137.07018, step = 39707 (52.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03986\n",
      "INFO:tensorflow:loss = 129.4279, step = 39807 (49.023 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94299\n",
      "INFO:tensorflow:loss = 81.226585, step = 39907 (51.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99729\n",
      "INFO:tensorflow:loss = 90.82385, step = 40007 (50.068 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9729\n",
      "INFO:tensorflow:loss = 106.12398, step = 40107 (50.687 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05946\n",
      "INFO:tensorflow:loss = 69.08267, step = 40207 (48.556 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92945\n",
      "INFO:tensorflow:loss = 133.5723, step = 40307 (51.828 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90994\n",
      "INFO:tensorflow:loss = 98.36992, step = 40407 (52.357 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93779\n",
      "INFO:tensorflow:loss = 90.62669, step = 40507 (51.606 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40549 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 97.597946.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-40549\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 286.40it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:52<00:00, 44.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.45000001788611516\n",
      "[test] Raw Hits@1: 32.06\n",
      "[test] Raw Hits@3: 53.080000000000005\n",
      "[test] Raw Hits@5: 60.74\n",
      "[test] Raw Hits@10: 69.02000000000001\n",
      "[test] Filtered MRR: 0.5819404725711662\n",
      "[test] Filtered Hits@1: 47.260000000000005\n",
      "[test] Filtered Hits@3: 65.84\n",
      "[test] Filtered Hits@5: 70.94\n",
      "[test] Filtered Hits@10: 76.9\n",
      "INFO:tensorflow:Epoch 44\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-40549\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 40550 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 104.50013, step = 40550\n",
      "INFO:tensorflow:global_step/sec: 1.89646\n",
      "INFO:tensorflow:loss = 70.175705, step = 40650 (52.731 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89836\n",
      "INFO:tensorflow:loss = 113.37696, step = 40750 (52.677 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00837\n",
      "INFO:tensorflow:loss = 94.139305, step = 40850 (49.792 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96164\n",
      "INFO:tensorflow:loss = 119.31904, step = 40950 (50.978 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94416\n",
      "INFO:tensorflow:loss = 72.968414, step = 41050 (51.436 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98252\n",
      "INFO:tensorflow:loss = 74.85371, step = 41150 (50.441 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00694\n",
      "INFO:tensorflow:loss = 99.68878, step = 41250 (49.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96922\n",
      "INFO:tensorflow:loss = 132.54453, step = 41350 (50.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97791\n",
      "INFO:tensorflow:loss = 97.18047, step = 41450 (50.559 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 41492 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 127.44574.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-41492\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 301.96it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4554070377706181\n",
      "[test] Raw Hits@1: 32.74\n",
      "[test] Raw Hits@3: 53.2\n",
      "[test] Raw Hits@5: 61.3\n",
      "[test] Raw Hits@10: 69.86\n",
      "[test] Filtered MRR: 0.5920946805499185\n",
      "[test] Filtered Hits@1: 48.480000000000004\n",
      "[test] Filtered Hits@3: 66.52\n",
      "[test] Filtered Hits@5: 71.74000000000001\n",
      "[test] Filtered Hits@10: 77.68\n",
      "INFO:tensorflow:Epoch 45\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-41492\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 41493 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 97.41497, step = 41493\n",
      "INFO:tensorflow:global_step/sec: 1.89543\n",
      "INFO:tensorflow:loss = 85.61416, step = 41593 (52.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98713\n",
      "INFO:tensorflow:loss = 76.637085, step = 41693 (50.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97397\n",
      "INFO:tensorflow:loss = 87.97828, step = 41793 (50.659 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0038\n",
      "INFO:tensorflow:loss = 112.121574, step = 41893 (49.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93674\n",
      "INFO:tensorflow:loss = 105.44754, step = 41993 (51.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96012\n",
      "INFO:tensorflow:loss = 113.78827, step = 42093 (51.017 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91673\n",
      "INFO:tensorflow:loss = 86.77613, step = 42193 (52.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94951\n",
      "INFO:tensorflow:loss = 135.1572, step = 42293 (51.295 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96193\n",
      "INFO:tensorflow:loss = 115.54622, step = 42393 (50.970 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 42435 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 106.75818.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-42435\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:18<00:00, 276.01it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.46161045258653394\n",
      "[test] Raw Hits@1: 33.300000000000004\n",
      "[test] Raw Hits@3: 54.24\n",
      "[test] Raw Hits@5: 61.419999999999995\n",
      "[test] Raw Hits@10: 69.94\n",
      "[test] Filtered MRR: 0.6031109493065242\n",
      "[test] Filtered Hits@1: 50.019999999999996\n",
      "[test] Filtered Hits@3: 67.62\n",
      "[test] Filtered Hits@5: 71.98\n",
      "[test] Filtered Hits@10: 78.28\n",
      "INFO:tensorflow:Epoch 46\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-42435\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 42436 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 82.91111, step = 42436\n",
      "INFO:tensorflow:global_step/sec: 1.95862\n",
      "INFO:tensorflow:loss = 107.28526, step = 42536 (51.058 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06702\n",
      "INFO:tensorflow:loss = 76.5744, step = 42636 (48.379 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09474\n",
      "INFO:tensorflow:loss = 100.40253, step = 42736 (47.738 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.06192\n",
      "INFO:tensorflow:loss = 84.190636, step = 42836 (48.498 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95359\n",
      "INFO:tensorflow:loss = 102.35701, step = 42936 (51.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.70927\n",
      "INFO:tensorflow:loss = 124.188446, step = 43036 (58.505 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.65361\n",
      "INFO:tensorflow:loss = 90.25844, step = 43136 (60.473 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91672\n",
      "INFO:tensorflow:loss = 82.735176, step = 43236 (52.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91736\n",
      "INFO:tensorflow:loss = 94.21832, step = 43336 (52.155 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 43378 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 80.5325.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-43378\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.59it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.46030205937924573\n",
      "[test] Raw Hits@1: 33.339999999999996\n",
      "[test] Raw Hits@3: 53.52\n",
      "[test] Raw Hits@5: 61.260000000000005\n",
      "[test] Raw Hits@10: 69.92\n",
      "[test] Filtered MRR: 0.6008739460770268\n",
      "[test] Filtered Hits@1: 49.74\n",
      "[test] Filtered Hits@3: 67.02\n",
      "[test] Filtered Hits@5: 72.36\n",
      "[test] Filtered Hits@10: 78.34\n",
      "INFO:tensorflow:Epoch 47\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-43378\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 43379 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 82.68355, step = 43379\n",
      "INFO:tensorflow:global_step/sec: 1.90141\n",
      "INFO:tensorflow:loss = 96.183975, step = 43479 (52.594 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92817\n",
      "INFO:tensorflow:loss = 135.7164, step = 43579 (51.863 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89272\n",
      "INFO:tensorflow:loss = 74.6162, step = 43679 (52.834 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93381\n",
      "INFO:tensorflow:loss = 86.00483, step = 43779 (51.711 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93903\n",
      "INFO:tensorflow:loss = 68.30415, step = 43879 (51.572 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91807\n",
      "INFO:tensorflow:loss = 76.973976, step = 43979 (52.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90434\n",
      "INFO:tensorflow:loss = 100.64827, step = 44079 (52.512 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93817\n",
      "INFO:tensorflow:loss = 104.31976, step = 44179 (51.595 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89889\n",
      "INFO:tensorflow:loss = 81.75583, step = 44279 (52.663 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 44321 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 83.81814.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-44321\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 296.51it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4632792534110614\n",
      "[test] Raw Hits@1: 33.6\n",
      "[test] Raw Hits@3: 53.800000000000004\n",
      "[test] Raw Hits@5: 61.5\n",
      "[test] Raw Hits@10: 71.06\n",
      "[test] Filtered MRR: 0.6054460459032792\n",
      "[test] Filtered Hits@1: 50.080000000000005\n",
      "[test] Filtered Hits@3: 67.4\n",
      "[test] Filtered Hits@5: 72.98\n",
      "[test] Filtered Hits@10: 79.52\n",
      "INFO:tensorflow:Epoch 48\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-44321\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 44322 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 66.32165, step = 44322\n",
      "INFO:tensorflow:global_step/sec: 1.90865\n",
      "INFO:tensorflow:loss = 53.681557, step = 44422 (52.394 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92919\n",
      "INFO:tensorflow:loss = 93.70595, step = 44522 (51.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92164\n",
      "INFO:tensorflow:loss = 96.71596, step = 44622 (52.039 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02019\n",
      "INFO:tensorflow:loss = 93.4524, step = 44722 (49.500 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04923\n",
      "INFO:tensorflow:loss = 108.689735, step = 44822 (48.799 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01371\n",
      "INFO:tensorflow:loss = 72.30205, step = 44922 (49.660 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93795\n",
      "INFO:tensorflow:loss = 56.752632, step = 45022 (51.600 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02264\n",
      "INFO:tensorflow:loss = 79.11967, step = 45122 (49.440 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92304\n",
      "INFO:tensorflow:loss = 124.70243, step = 45222 (52.001 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 45264 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 96.7863.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-45264\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.57it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4622729741894029\n",
      "[test] Raw Hits@1: 33.28\n",
      "[test] Raw Hits@3: 53.959999999999994\n",
      "[test] Raw Hits@5: 61.739999999999995\n",
      "[test] Raw Hits@10: 70.26\n",
      "[test] Filtered MRR: 0.6070305890201549\n",
      "[test] Filtered Hits@1: 50.13999999999999\n",
      "[test] Filtered Hits@3: 67.82000000000001\n",
      "[test] Filtered Hits@5: 72.98\n",
      "[test] Filtered Hits@10: 79.08\n",
      "INFO:tensorflow:Epoch 49\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-45264\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 45265 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 71.67327, step = 45265\n",
      "INFO:tensorflow:global_step/sec: 1.95076\n",
      "INFO:tensorflow:loss = 84.58948, step = 45365 (51.263 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9528\n",
      "INFO:tensorflow:loss = 67.389854, step = 45465 (51.208 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94251\n",
      "INFO:tensorflow:loss = 65.94957, step = 45565 (51.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93484\n",
      "INFO:tensorflow:loss = 68.33284, step = 45665 (51.684 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89574\n",
      "INFO:tensorflow:loss = 101.68442, step = 45765 (52.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93707\n",
      "INFO:tensorflow:loss = 62.322445, step = 45865 (51.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96137\n",
      "INFO:tensorflow:loss = 113.22363, step = 45965 (50.985 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94515\n",
      "INFO:tensorflow:loss = 69.52799, step = 46065 (51.410 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07699\n",
      "INFO:tensorflow:loss = 94.60585, step = 46165 (48.147 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 46207 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 76.87476.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-46207\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 301.16it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:43<00:00, 48.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4631276895638431\n",
      "[test] Raw Hits@1: 33.48\n",
      "[test] Raw Hits@3: 53.68000000000001\n",
      "[test] Raw Hits@5: 61.78\n",
      "[test] Raw Hits@10: 70.62\n",
      "[test] Filtered MRR: 0.6050799184941159\n",
      "[test] Filtered Hits@1: 50.1\n",
      "[test] Filtered Hits@3: 67.34\n",
      "[test] Filtered Hits@5: 72.98\n",
      "[test] Filtered Hits@10: 78.97999999999999\n",
      "INFO:tensorflow:Epoch 50\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-46207\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 46208 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 80.983665, step = 46208\n",
      "INFO:tensorflow:global_step/sec: 2.01839\n",
      "INFO:tensorflow:loss = 57.307793, step = 46308 (49.546 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09827\n",
      "INFO:tensorflow:loss = 135.40999, step = 46408 (47.658 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94238\n",
      "INFO:tensorflow:loss = 82.18518, step = 46508 (51.484 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07828\n",
      "INFO:tensorflow:loss = 123.455444, step = 46608 (48.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07086\n",
      "INFO:tensorflow:loss = 108.74498, step = 46708 (48.289 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19279\n",
      "INFO:tensorflow:loss = 125.5567, step = 46808 (45.604 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.15727\n",
      "INFO:tensorflow:loss = 58.713005, step = 46908 (46.355 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.15898\n",
      "INFO:tensorflow:loss = 96.863846, step = 47008 (46.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18942\n",
      "INFO:tensorflow:loss = 59.65511, step = 47108 (45.674 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 47150 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 89.44145.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-47150\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 299.96it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:42<00:00, 48.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4667002818541142\n",
      "[test] Raw Hits@1: 34.18\n",
      "[test] Raw Hits@3: 54.1\n",
      "[test] Raw Hits@5: 61.88\n",
      "[test] Raw Hits@10: 70.62\n",
      "[test] Filtered MRR: 0.6101556567006253\n",
      "[test] Filtered Hits@1: 50.82\n",
      "[test] Filtered Hits@3: 67.75999999999999\n",
      "[test] Filtered Hits@5: 73.2\n",
      "[test] Filtered Hits@10: 78.58000000000001\n",
      "INFO:tensorflow:Epoch 51\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-47150\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 47151 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 53.595234, step = 47151\n",
      "INFO:tensorflow:global_step/sec: 1.86339\n",
      "INFO:tensorflow:loss = 82.682655, step = 47251 (53.667 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95589\n",
      "INFO:tensorflow:loss = 66.70686, step = 47351 (51.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98296\n",
      "INFO:tensorflow:loss = 53.072906, step = 47451 (50.429 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96714\n",
      "INFO:tensorflow:loss = 77.22818, step = 47551 (50.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03756\n",
      "INFO:tensorflow:loss = 85.235214, step = 47651 (49.078 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04195\n",
      "INFO:tensorflow:loss = 80.90431, step = 47751 (48.973 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96191\n",
      "INFO:tensorflow:loss = 101.22462, step = 47851 (50.971 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94411\n",
      "INFO:tensorflow:loss = 100.25336, step = 47951 (51.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95303\n",
      "INFO:tensorflow:loss = 92.48304, step = 48051 (51.202 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 48093 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 67.439316.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-48093\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 294.33it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.46431642191241573\n",
      "[test] Raw Hits@1: 33.62\n",
      "[test] Raw Hits@3: 54.279999999999994\n",
      "[test] Raw Hits@5: 61.760000000000005\n",
      "[test] Raw Hits@10: 70.66\n",
      "[test] Filtered MRR: 0.6096424071218518\n",
      "[test] Filtered Hits@1: 50.72\n",
      "[test] Filtered Hits@3: 67.74\n",
      "[test] Filtered Hits@5: 73.28\n",
      "[test] Filtered Hits@10: 79.25999999999999\n",
      "INFO:tensorflow:Epoch 52\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-48093\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 48094 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 70.57683, step = 48094\n",
      "INFO:tensorflow:global_step/sec: 1.98102\n",
      "INFO:tensorflow:loss = 85.79223, step = 48194 (50.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85233\n",
      "INFO:tensorflow:loss = 70.51819, step = 48294 (53.986 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92199\n",
      "INFO:tensorflow:loss = 91.414406, step = 48394 (52.030 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00952\n",
      "INFO:tensorflow:loss = 90.80206, step = 48494 (49.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.989\n",
      "INFO:tensorflow:loss = 90.40026, step = 48594 (50.277 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05777\n",
      "INFO:tensorflow:loss = 78.166336, step = 48694 (48.596 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0801\n",
      "INFO:tensorflow:loss = 60.83821, step = 48794 (48.075 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9439\n",
      "INFO:tensorflow:loss = 98.2693, step = 48894 (51.443 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02414\n",
      "INFO:tensorflow:loss = 96.56011, step = 48994 (49.404 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 49036 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 85.68146.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-49036\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 294.48it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.46491302509658944\n",
      "[test] Raw Hits@1: 33.52\n",
      "[test] Raw Hits@3: 53.92\n",
      "[test] Raw Hits@5: 62.06\n",
      "[test] Raw Hits@10: 70.78\n",
      "[test] Filtered MRR: 0.6095493032407036\n",
      "[test] Filtered Hits@1: 50.339999999999996\n",
      "[test] Filtered Hits@3: 68.2\n",
      "[test] Filtered Hits@5: 73.5\n",
      "[test] Filtered Hits@10: 79.62\n",
      "INFO:tensorflow:Epoch 53\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-49036\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 49037 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 92.4159, step = 49037\n",
      "INFO:tensorflow:global_step/sec: 2.00748\n",
      "INFO:tensorflow:loss = 59.762993, step = 49137 (49.815 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9969\n",
      "INFO:tensorflow:loss = 92.92636, step = 49237 (50.078 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.86506\n",
      "INFO:tensorflow:loss = 62.210144, step = 49337 (53.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87625\n",
      "INFO:tensorflow:loss = 82.81271, step = 49437 (53.297 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89723\n",
      "INFO:tensorflow:loss = 75.930504, step = 49537 (52.708 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00063\n",
      "INFO:tensorflow:loss = 71.40821, step = 49637 (49.984 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90462\n",
      "INFO:tensorflow:loss = 65.803856, step = 49737 (52.504 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82888\n",
      "INFO:tensorflow:loss = 59.629253, step = 49837 (54.678 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97671\n",
      "INFO:tensorflow:loss = 89.8273, step = 49937 (50.589 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 49979 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 72.95901.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-49979\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 279.91it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:52<00:00, 44.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.46342488155164957\n",
      "[test] Raw Hits@1: 33.4\n",
      "[test] Raw Hits@3: 53.959999999999994\n",
      "[test] Raw Hits@5: 61.839999999999996\n",
      "[test] Raw Hits@10: 70.39999999999999\n",
      "[test] Filtered MRR: 0.6112617448341389\n",
      "[test] Filtered Hits@1: 50.580000000000005\n",
      "[test] Filtered Hits@3: 68.47999999999999\n",
      "[test] Filtered Hits@5: 73.5\n",
      "[test] Filtered Hits@10: 79.28\n",
      "INFO:tensorflow:Epoch 54\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-49979\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 49980 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 77.049904, step = 49980\n",
      "INFO:tensorflow:global_step/sec: 1.95437\n",
      "INFO:tensorflow:loss = 81.6063, step = 50080 (51.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87612\n",
      "INFO:tensorflow:loss = 84.870705, step = 50180 (53.302 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89328\n",
      "INFO:tensorflow:loss = 81.86813, step = 50280 (52.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.80778\n",
      "INFO:tensorflow:loss = 74.22526, step = 50380 (55.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.81739\n",
      "INFO:tensorflow:loss = 89.89136, step = 50480 (55.024 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.86722\n",
      "INFO:tensorflow:loss = 79.814835, step = 50580 (53.556 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82704\n",
      "INFO:tensorflow:loss = 68.39139, step = 50680 (54.733 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94155\n",
      "INFO:tensorflow:loss = 85.402664, step = 50780 (51.505 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94156\n",
      "INFO:tensorflow:loss = 81.87214, step = 50880 (51.505 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 50922 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 88.13867.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-50922\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:18<00:00, 274.37it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.46680486080904554\n",
      "[test] Raw Hits@1: 33.98\n",
      "[test] Raw Hits@3: 54.279999999999994\n",
      "[test] Raw Hits@5: 62.22\n",
      "[test] Raw Hits@10: 70.78\n",
      "[test] Filtered MRR: 0.6148233795714797\n",
      "[test] Filtered Hits@1: 51.080000000000005\n",
      "[test] Filtered Hits@3: 69.0\n",
      "[test] Filtered Hits@5: 74.08\n",
      "[test] Filtered Hits@10: 79.46\n",
      "INFO:tensorflow:Epoch 55\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-50922\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 50923 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 74.43913, step = 50923\n",
      "INFO:tensorflow:global_step/sec: 1.85305\n",
      "INFO:tensorflow:loss = 59.0954, step = 51023 (53.966 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00546\n",
      "INFO:tensorflow:loss = 75.278114, step = 51123 (49.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97574\n",
      "INFO:tensorflow:loss = 80.845604, step = 51223 (50.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.80846\n",
      "INFO:tensorflow:loss = 59.437233, step = 51323 (55.296 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96169\n",
      "INFO:tensorflow:loss = 105.79785, step = 51423 (50.976 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04411\n",
      "INFO:tensorflow:loss = 79.96623, step = 51523 (48.921 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89183\n",
      "INFO:tensorflow:loss = 78.81121, step = 51623 (52.859 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88747\n",
      "INFO:tensorflow:loss = 72.958984, step = 51723 (52.982 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09108\n",
      "INFO:tensorflow:loss = 92.87669, step = 51823 (47.822 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 51865 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 76.08275.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-51865\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 278.64it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4731547240070651\n",
      "[test] Raw Hits@1: 34.660000000000004\n",
      "[test] Raw Hits@3: 54.6\n",
      "[test] Raw Hits@5: 63.13999999999999\n",
      "[test] Raw Hits@10: 71.67999999999999\n",
      "[test] Filtered MRR: 0.6254116409701335\n",
      "[test] Filtered Hits@1: 52.35999999999999\n",
      "[test] Filtered Hits@3: 69.19999999999999\n",
      "[test] Filtered Hits@5: 75.06\n",
      "[test] Filtered Hits@10: 79.9\n",
      "INFO:tensorflow:Epoch 56\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-51865\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 51866 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 61.022438, step = 51866\n",
      "INFO:tensorflow:global_step/sec: 1.86716\n",
      "INFO:tensorflow:loss = 100.31596, step = 51966 (53.558 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.80764\n",
      "INFO:tensorflow:loss = 105.74329, step = 52066 (55.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98764\n",
      "INFO:tensorflow:loss = 59.669426, step = 52166 (50.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90315\n",
      "INFO:tensorflow:loss = 65.2392, step = 52266 (52.546 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91121\n",
      "INFO:tensorflow:loss = 70.899414, step = 52366 (52.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87903\n",
      "INFO:tensorflow:loss = 59.351303, step = 52466 (53.217 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03928\n",
      "INFO:tensorflow:loss = 53.08715, step = 52566 (49.037 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97177\n",
      "INFO:tensorflow:loss = 45.4363, step = 52666 (50.716 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94542\n",
      "INFO:tensorflow:loss = 72.172844, step = 52766 (51.403 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 52808 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 69.27156.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-52808\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 301.05it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:47<00:00, 46.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4730843747173134\n",
      "[test] Raw Hits@1: 35.44\n",
      "[test] Raw Hits@3: 53.800000000000004\n",
      "[test] Raw Hits@5: 61.519999999999996\n",
      "[test] Raw Hits@10: 70.34\n",
      "[test] Filtered MRR: 0.6112014038899655\n",
      "[test] Filtered Hits@1: 51.480000000000004\n",
      "[test] Filtered Hits@3: 67.30000000000001\n",
      "[test] Filtered Hits@5: 72.28\n",
      "[test] Filtered Hits@10: 78.36\n",
      "INFO:tensorflow:Epoch 57\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-52808\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 52809 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 64.42834, step = 52809\n",
      "INFO:tensorflow:global_step/sec: 2.09712\n",
      "INFO:tensorflow:loss = 46.04133, step = 52909 (47.686 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0094\n",
      "INFO:tensorflow:loss = 70.375305, step = 53009 (49.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02052\n",
      "INFO:tensorflow:loss = 75.31942, step = 53109 (49.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01862\n",
      "INFO:tensorflow:loss = 83.11007, step = 53209 (49.539 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.08989\n",
      "INFO:tensorflow:loss = 108.77414, step = 53309 (47.849 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94667\n",
      "INFO:tensorflow:loss = 52.906517, step = 53409 (51.370 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0463\n",
      "INFO:tensorflow:loss = 91.82096, step = 53509 (48.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9717\n",
      "INFO:tensorflow:loss = 70.899055, step = 53609 (50.718 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82649\n",
      "INFO:tensorflow:loss = 108.57286, step = 53709 (54.750 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 53751 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 69.95989.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-53751\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 289.18it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:52<00:00, 44.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4739985303884114\n",
      "[test] Raw Hits@1: 34.88\n",
      "[test] Raw Hits@3: 54.94\n",
      "[test] Raw Hits@5: 62.82\n",
      "[test] Raw Hits@10: 71.0\n",
      "[test] Filtered MRR: 0.6207634702453337\n",
      "[test] Filtered Hits@1: 51.839999999999996\n",
      "[test] Filtered Hits@3: 69.04\n",
      "[test] Filtered Hits@5: 73.94\n",
      "[test] Filtered Hits@10: 79.66\n",
      "INFO:tensorflow:Epoch 58\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-53751\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 53752 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 78.30938, step = 53752\n",
      "INFO:tensorflow:global_step/sec: 1.86672\n",
      "INFO:tensorflow:loss = 81.14526, step = 53852 (53.571 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89349\n",
      "INFO:tensorflow:loss = 54.12139, step = 53952 (52.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97942\n",
      "INFO:tensorflow:loss = 60.754616, step = 54052 (50.520 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93969\n",
      "INFO:tensorflow:loss = 53.7688, step = 54152 (51.553 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91517\n",
      "INFO:tensorflow:loss = 81.02601, step = 54252 (52.215 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.85462\n",
      "INFO:tensorflow:loss = 99.074, step = 54352 (53.920 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.8213\n",
      "INFO:tensorflow:loss = 99.12036, step = 54452 (54.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92503\n",
      "INFO:tensorflow:loss = 90.65853, step = 54552 (51.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90175\n",
      "INFO:tensorflow:loss = 110.232285, step = 54652 (52.583 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 54694 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 74.86381.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-54694\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 291.55it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:46<00:00, 46.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4705301047152818\n",
      "[test] Raw Hits@1: 34.160000000000004\n",
      "[test] Raw Hits@3: 54.67999999999999\n",
      "[test] Raw Hits@5: 62.980000000000004\n",
      "[test] Raw Hits@10: 71.32\n",
      "[test] Filtered MRR: 0.6188099256895534\n",
      "[test] Filtered Hits@1: 51.42\n",
      "[test] Filtered Hits@3: 68.96\n",
      "[test] Filtered Hits@5: 74.56\n",
      "[test] Filtered Hits@10: 79.86\n",
      "INFO:tensorflow:Epoch 59\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-54694\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 54695 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 56.51614, step = 54695\n",
      "INFO:tensorflow:global_step/sec: 1.89511\n",
      "INFO:tensorflow:loss = 57.697838, step = 54795 (52.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88298\n",
      "INFO:tensorflow:loss = 83.07871, step = 54895 (53.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97184\n",
      "INFO:tensorflow:loss = 67.13891, step = 54995 (50.714 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87842\n",
      "INFO:tensorflow:loss = 76.62644, step = 55095 (53.236 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03626\n",
      "INFO:tensorflow:loss = 75.59258, step = 55195 (49.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03967\n",
      "INFO:tensorflow:loss = 78.95373, step = 55295 (49.028 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00275\n",
      "INFO:tensorflow:loss = 73.84395, step = 55395 (49.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05085\n",
      "INFO:tensorflow:loss = 64.273224, step = 55495 (48.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92168\n",
      "INFO:tensorflow:loss = 51.022255, step = 55595 (52.038 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 55637 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 63.726177.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-55637\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 291.83it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.47737045939371003\n",
      "[test] Raw Hits@1: 35.4\n",
      "[test] Raw Hits@3: 55.120000000000005\n",
      "[test] Raw Hits@5: 63.04\n",
      "[test] Raw Hits@10: 71.38\n",
      "[test] Filtered MRR: 0.6276425134271827\n",
      "[test] Filtered Hits@1: 52.839999999999996\n",
      "[test] Filtered Hits@3: 69.58\n",
      "[test] Filtered Hits@5: 74.5\n",
      "[test] Filtered Hits@10: 79.82000000000001\n",
      "INFO:tensorflow:Epoch 60\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-55637\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 55638 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 56.82495, step = 55638\n",
      "INFO:tensorflow:global_step/sec: 1.97496\n",
      "INFO:tensorflow:loss = 55.11692, step = 55738 (50.635 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02092\n",
      "INFO:tensorflow:loss = 54.25243, step = 55838 (49.482 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.11603\n",
      "INFO:tensorflow:loss = 50.90029, step = 55938 (47.259 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.10689\n",
      "INFO:tensorflow:loss = 59.983498, step = 56038 (47.463 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01037\n",
      "INFO:tensorflow:loss = 77.73042, step = 56138 (49.742 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05037\n",
      "INFO:tensorflow:loss = 64.88416, step = 56238 (48.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99705\n",
      "INFO:tensorflow:loss = 96.915146, step = 56338 (50.074 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.09701\n",
      "INFO:tensorflow:loss = 70.275604, step = 56438 (47.688 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96547\n",
      "INFO:tensorflow:loss = 72.95611, step = 56538 (50.878 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 56580 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 66.306366.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-56580\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.09it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.47322597491003626\n",
      "[test] Raw Hits@1: 34.72\n",
      "[test] Raw Hits@3: 54.67999999999999\n",
      "[test] Raw Hits@5: 62.660000000000004\n",
      "[test] Raw Hits@10: 71.1\n",
      "[test] Filtered MRR: 0.6185601478775637\n",
      "[test] Filtered Hits@1: 51.66\n",
      "[test] Filtered Hits@3: 68.60000000000001\n",
      "[test] Filtered Hits@5: 73.83999999999999\n",
      "[test] Filtered Hits@10: 79.5\n",
      "INFO:tensorflow:Epoch 61\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-56580\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 56581 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 54.7906, step = 56581\n",
      "INFO:tensorflow:global_step/sec: 1.83928\n",
      "INFO:tensorflow:loss = 72.21041, step = 56681 (54.370 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90667\n",
      "INFO:tensorflow:loss = 62.72625, step = 56781 (52.447 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87454\n",
      "INFO:tensorflow:loss = 72.27766, step = 56881 (53.346 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88543\n",
      "INFO:tensorflow:loss = 95.86487, step = 56981 (53.038 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.8901\n",
      "INFO:tensorflow:loss = 85.173096, step = 57081 (52.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93322\n",
      "INFO:tensorflow:loss = 73.133705, step = 57181 (51.727 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92837\n",
      "INFO:tensorflow:loss = 69.00216, step = 57281 (51.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88378\n",
      "INFO:tensorflow:loss = 83.687546, step = 57381 (53.085 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.52767\n",
      "INFO:tensorflow:loss = 72.52311, step = 57481 (65.460 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 57523 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 83.85997.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-57523\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 293.06it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:55<00:00, 43.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.47877384290932495\n",
      "[test] Raw Hits@1: 35.46\n",
      "[test] Raw Hits@3: 55.2\n",
      "[test] Raw Hits@5: 63.019999999999996\n",
      "[test] Raw Hits@10: 71.34\n",
      "[test] Filtered MRR: 0.6300805022420226\n",
      "[test] Filtered Hits@1: 53.300000000000004\n",
      "[test] Filtered Hits@3: 69.46\n",
      "[test] Filtered Hits@5: 74.52\n",
      "[test] Filtered Hits@10: 79.74\n",
      "INFO:tensorflow:Epoch 62\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-57523\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 57524 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 59.407196, step = 57524\n",
      "INFO:tensorflow:global_step/sec: 1.17506\n",
      "INFO:tensorflow:loss = 47.585514, step = 57624 (85.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.24026\n",
      "INFO:tensorflow:loss = 64.87566, step = 57724 (80.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.3128\n",
      "INFO:tensorflow:loss = 58.43976, step = 57824 (76.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.51469\n",
      "INFO:tensorflow:loss = 69.30326, step = 57924 (66.022 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.45665\n",
      "INFO:tensorflow:loss = 91.59662, step = 58024 (68.649 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.30305\n",
      "INFO:tensorflow:loss = 84.489784, step = 58124 (76.743 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.6496\n",
      "INFO:tensorflow:loss = 78.901184, step = 58224 (60.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.72339\n",
      "INFO:tensorflow:loss = 38.01814, step = 58324 (58.024 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 58374 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 1.85841\n",
      "INFO:tensorflow:loss = 79.95872, step = 58424 (53.810 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 58466 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 87.79307.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-58466\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.78it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:54<00:00, 43.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4800693542583153\n",
      "[test] Raw Hits@1: 35.54\n",
      "[test] Raw Hits@3: 55.52\n",
      "[test] Raw Hits@5: 63.080000000000005\n",
      "[test] Raw Hits@10: 71.6\n",
      "[test] Filtered MRR: 0.6306140085492193\n",
      "[test] Filtered Hits@1: 53.36\n",
      "[test] Filtered Hits@3: 69.64\n",
      "[test] Filtered Hits@5: 74.5\n",
      "[test] Filtered Hits@10: 80.17999999999999\n",
      "INFO:tensorflow:Epoch 63\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-58466\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 58467 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 81.14758, step = 58467\n",
      "INFO:tensorflow:global_step/sec: 1.86881\n",
      "INFO:tensorflow:loss = 54.1589, step = 58567 (53.511 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89823\n",
      "INFO:tensorflow:loss = 68.47879, step = 58667 (52.681 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96698\n",
      "INFO:tensorflow:loss = 74.6203, step = 58767 (50.840 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.02752\n",
      "INFO:tensorflow:loss = 69.4259, step = 58867 (49.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92347\n",
      "INFO:tensorflow:loss = 97.101494, step = 58967 (51.989 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98361\n",
      "INFO:tensorflow:loss = 52.282543, step = 59067 (50.413 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89872\n",
      "INFO:tensorflow:loss = 62.24278, step = 59167 (52.667 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98143\n",
      "INFO:tensorflow:loss = 85.051636, step = 59267 (50.469 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.54267\n",
      "INFO:tensorflow:loss = 75.67483, step = 59367 (64.823 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 59409 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 78.98826.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-59409\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:18<00:00, 264.64it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:52<00:00, 44.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.47828317082883204\n",
      "[test] Raw Hits@1: 34.88\n",
      "[test] Raw Hits@3: 55.76\n",
      "[test] Raw Hits@5: 63.5\n",
      "[test] Raw Hits@10: 72.11999999999999\n",
      "[test] Filtered MRR: 0.6353506271511131\n",
      "[test] Filtered Hits@1: 53.459999999999994\n",
      "[test] Filtered Hits@3: 70.28\n",
      "[test] Filtered Hits@5: 75.22\n",
      "[test] Filtered Hits@10: 80.80000000000001\n",
      "INFO:tensorflow:Epoch 64\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-59409\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 59410 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 66.22578, step = 59410\n",
      "INFO:tensorflow:global_step/sec: 1.79835\n",
      "INFO:tensorflow:loss = 47.28069, step = 59510 (55.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.84487\n",
      "INFO:tensorflow:loss = 59.552017, step = 59610 (54.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91331\n",
      "INFO:tensorflow:loss = 45.967915, step = 59710 (52.265 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92342\n",
      "INFO:tensorflow:loss = 55.284782, step = 59810 (51.992 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88309\n",
      "INFO:tensorflow:loss = 84.77603, step = 59910 (53.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90271\n",
      "INFO:tensorflow:loss = 63.106346, step = 60010 (52.557 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98675\n",
      "INFO:tensorflow:loss = 82.32076, step = 60110 (50.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98074\n",
      "INFO:tensorflow:loss = 58.540794, step = 60210 (50.487 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.78319\n",
      "INFO:tensorflow:loss = 55.414658, step = 60310 (56.079 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 60352 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 62.96284.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-60352\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:19<00:00, 260.89it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:56<00:00, 42.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.47723581658401437\n",
      "[test] Raw Hits@1: 35.0\n",
      "[test] Raw Hits@3: 55.42\n",
      "[test] Raw Hits@5: 63.13999999999999\n",
      "[test] Raw Hits@10: 71.26\n",
      "[test] Filtered MRR: 0.628328919263018\n",
      "[test] Filtered Hits@1: 52.400000000000006\n",
      "[test] Filtered Hits@3: 70.52000000000001\n",
      "[test] Filtered Hits@5: 75.1\n",
      "[test] Filtered Hits@10: 80.06\n",
      "INFO:tensorflow:Epoch 65\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-60352\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 60353 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 40.407112, step = 60353\n",
      "INFO:tensorflow:global_step/sec: 1.75345\n",
      "INFO:tensorflow:loss = 53.244648, step = 60453 (57.033 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.70959\n",
      "INFO:tensorflow:loss = 48.061146, step = 60553 (58.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.76554\n",
      "INFO:tensorflow:loss = 66.823105, step = 60653 (56.640 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.81213\n",
      "INFO:tensorflow:loss = 68.80378, step = 60753 (55.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88786\n",
      "INFO:tensorflow:loss = 80.36554, step = 60853 (52.969 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.79615\n",
      "INFO:tensorflow:loss = 62.130714, step = 60953 (55.675 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89137\n",
      "INFO:tensorflow:loss = 64.081154, step = 61053 (52.872 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.83445\n",
      "INFO:tensorflow:loss = 59.535988, step = 61153 (54.512 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.77753\n",
      "INFO:tensorflow:loss = 101.61902, step = 61253 (56.258 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 61295 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 55.22449.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-61295\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 296.04it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:53<00:00, 44.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4785233113461105\n",
      "[test] Raw Hits@1: 35.22\n",
      "[test] Raw Hits@3: 55.620000000000005\n",
      "[test] Raw Hits@5: 63.36000000000001\n",
      "[test] Raw Hits@10: 71.41999999999999\n",
      "[test] Filtered MRR: 0.6333353557073444\n",
      "[test] Filtered Hits@1: 53.32\n",
      "[test] Filtered Hits@3: 70.14\n",
      "[test] Filtered Hits@5: 75.5\n",
      "[test] Filtered Hits@10: 80.36\n",
      "INFO:tensorflow:Epoch 66\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-61295\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 61296 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 55.229187, step = 61296\n",
      "INFO:tensorflow:global_step/sec: 1.88784\n",
      "INFO:tensorflow:loss = 52.572277, step = 61396 (52.972 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95411\n",
      "INFO:tensorflow:loss = 61.22037, step = 61496 (51.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94772\n",
      "INFO:tensorflow:loss = 48.555943, step = 61596 (51.342 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90197\n",
      "INFO:tensorflow:loss = 91.87368, step = 61696 (52.578 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98139\n",
      "INFO:tensorflow:loss = 89.000305, step = 61796 (50.468 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05129\n",
      "INFO:tensorflow:loss = 40.454826, step = 61896 (48.750 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.95758\n",
      "INFO:tensorflow:loss = 60.475613, step = 61996 (51.083 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92372\n",
      "INFO:tensorflow:loss = 60.874622, step = 62096 (51.982 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03716\n",
      "INFO:tensorflow:loss = 77.69897, step = 62196 (49.088 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 62238 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 85.718994.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-62238\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.22it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48105567957831347\n",
      "[test] Raw Hits@1: 35.839999999999996\n",
      "[test] Raw Hits@3: 54.800000000000004\n",
      "[test] Raw Hits@5: 63.260000000000005\n",
      "[test] Raw Hits@10: 71.8\n",
      "[test] Filtered MRR: 0.6329414019154016\n",
      "[test] Filtered Hits@1: 53.54\n",
      "[test] Filtered Hits@3: 70.04\n",
      "[test] Filtered Hits@5: 74.6\n",
      "[test] Filtered Hits@10: 80.28\n",
      "INFO:tensorflow:Epoch 67\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-62238\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 62239 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 65.203064, step = 62239\n",
      "INFO:tensorflow:global_step/sec: 1.84682\n",
      "INFO:tensorflow:loss = 50.359848, step = 62339 (54.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96647\n",
      "INFO:tensorflow:loss = 79.15603, step = 62439 (50.853 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82503\n",
      "INFO:tensorflow:loss = 67.05325, step = 62539 (54.793 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96339\n",
      "INFO:tensorflow:loss = 58.984627, step = 62639 (50.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90419\n",
      "INFO:tensorflow:loss = 42.499763, step = 62739 (52.515 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82447\n",
      "INFO:tensorflow:loss = 62.82328, step = 62839 (54.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98404\n",
      "INFO:tensorflow:loss = 61.562927, step = 62939 (50.402 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93013\n",
      "INFO:tensorflow:loss = 80.76956, step = 63039 (51.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9256\n",
      "INFO:tensorflow:loss = 73.99332, step = 63139 (51.932 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 63181 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 82.036835.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-63181\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 290.29it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:50<00:00, 45.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4784159722559394\n",
      "[test] Raw Hits@1: 35.52\n",
      "[test] Raw Hits@3: 54.72\n",
      "[test] Raw Hits@5: 62.82\n",
      "[test] Raw Hits@10: 71.38\n",
      "[test] Filtered MRR: 0.6259940228337789\n",
      "[test] Filtered Hits@1: 52.76\n",
      "[test] Filtered Hits@3: 69.04\n",
      "[test] Filtered Hits@5: 74.42\n",
      "[test] Filtered Hits@10: 80.32000000000001\n",
      "INFO:tensorflow:Epoch 68\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-63181\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 63182 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 49.69077, step = 63182\n",
      "INFO:tensorflow:global_step/sec: 1.84865\n",
      "INFO:tensorflow:loss = 63.51252, step = 63282 (54.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91128\n",
      "INFO:tensorflow:loss = 89.760864, step = 63382 (52.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89242\n",
      "INFO:tensorflow:loss = 71.35937, step = 63482 (52.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93312\n",
      "INFO:tensorflow:loss = 62.213448, step = 63582 (51.730 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94854\n",
      "INFO:tensorflow:loss = 57.70481, step = 63682 (51.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88842\n",
      "INFO:tensorflow:loss = 44.10739, step = 63782 (52.955 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91868\n",
      "INFO:tensorflow:loss = 42.7217, step = 63882 (52.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94021\n",
      "INFO:tensorflow:loss = 70.537125, step = 63982 (51.541 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92702\n",
      "INFO:tensorflow:loss = 82.491455, step = 64082 (51.894 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 64124 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 75.304085.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-64124\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 288.05it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.47764802483173063\n",
      "[test] Raw Hits@1: 35.22\n",
      "[test] Raw Hits@3: 55.16\n",
      "[test] Raw Hits@5: 63.04\n",
      "[test] Raw Hits@10: 71.96000000000001\n",
      "[test] Filtered MRR: 0.6315672020279751\n",
      "[test] Filtered Hits@1: 53.26\n",
      "[test] Filtered Hits@3: 69.96\n",
      "[test] Filtered Hits@5: 75.16000000000001\n",
      "[test] Filtered Hits@10: 80.76\n",
      "INFO:tensorflow:Epoch 69\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-64124\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 64125 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 81.113945, step = 64125\n",
      "INFO:tensorflow:global_step/sec: 1.90216\n",
      "INFO:tensorflow:loss = 59.169476, step = 64225 (52.573 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05092\n",
      "INFO:tensorflow:loss = 49.71805, step = 64325 (48.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.12139\n",
      "INFO:tensorflow:loss = 47.5347, step = 64425 (47.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05242\n",
      "INFO:tensorflow:loss = 43.93233, step = 64525 (48.723 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0627\n",
      "INFO:tensorflow:loss = 70.62815, step = 64625 (48.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00531\n",
      "INFO:tensorflow:loss = 57.92785, step = 64725 (49.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82051\n",
      "INFO:tensorflow:loss = 69.96757, step = 64825 (54.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.94212\n",
      "INFO:tensorflow:loss = 78.27397, step = 64925 (51.490 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91492\n",
      "INFO:tensorflow:loss = 63.0841, step = 65025 (52.222 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 65067 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.527306.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-65067\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 294.56it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:52<00:00, 44.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48363718251233306\n",
      "[test] Raw Hits@1: 36.08\n",
      "[test] Raw Hits@3: 55.279999999999994\n",
      "[test] Raw Hits@5: 63.56\n",
      "[test] Raw Hits@10: 72.32\n",
      "[test] Filtered MRR: 0.6421297307648746\n",
      "[test] Filtered Hits@1: 54.559999999999995\n",
      "[test] Filtered Hits@3: 70.56\n",
      "[test] Filtered Hits@5: 76.06\n",
      "[test] Filtered Hits@10: 81.12\n",
      "INFO:tensorflow:Epoch 70\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-65067\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 65068 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 49.82701, step = 65068\n",
      "INFO:tensorflow:global_step/sec: 1.7982\n",
      "INFO:tensorflow:loss = 81.5714, step = 65168 (55.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90903\n",
      "INFO:tensorflow:loss = 59.83705, step = 65268 (52.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88676\n",
      "INFO:tensorflow:loss = 51.981155, step = 65368 (53.000 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90266\n",
      "INFO:tensorflow:loss = 55.1645, step = 65468 (52.558 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03971\n",
      "INFO:tensorflow:loss = 62.85694, step = 65568 (49.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96716\n",
      "INFO:tensorflow:loss = 53.552177, step = 65668 (50.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93813\n",
      "INFO:tensorflow:loss = 98.83155, step = 65768 (51.596 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.84013\n",
      "INFO:tensorflow:loss = 54.687435, step = 65868 (54.344 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.84057\n",
      "INFO:tensorflow:loss = 96.72345, step = 65968 (54.331 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 66010 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 54.40948.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-66010\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:18<00:00, 270.92it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48234514616320195\n",
      "[test] Raw Hits@1: 35.839999999999996\n",
      "[test] Raw Hits@3: 55.400000000000006\n",
      "[test] Raw Hits@5: 63.739999999999995\n",
      "[test] Raw Hits@10: 72.1\n",
      "[test] Filtered MRR: 0.639412185295497\n",
      "[test] Filtered Hits@1: 54.14\n",
      "[test] Filtered Hits@3: 70.66\n",
      "[test] Filtered Hits@5: 75.68\n",
      "[test] Filtered Hits@10: 81.2\n",
      "INFO:tensorflow:Epoch 71\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-66010\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 66011 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 55.351578, step = 66011\n",
      "INFO:tensorflow:global_step/sec: 2.0038\n",
      "INFO:tensorflow:loss = 41.85927, step = 66111 (49.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.9096\n",
      "INFO:tensorflow:loss = 43.94875, step = 66211 (52.367 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.82782\n",
      "INFO:tensorflow:loss = 57.644234, step = 66311 (54.712 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92603\n",
      "INFO:tensorflow:loss = 87.27588, step = 66411 (51.918 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.80441\n",
      "INFO:tensorflow:loss = 53.20056, step = 66511 (55.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87878\n",
      "INFO:tensorflow:loss = 53.39869, step = 66611 (53.226 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.7803\n",
      "INFO:tensorflow:loss = 56.52972, step = 66711 (56.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87522\n",
      "INFO:tensorflow:loss = 47.970764, step = 66811 (53.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.81428\n",
      "INFO:tensorflow:loss = 46.639214, step = 66911 (55.119 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 66953 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 55.574398.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-66953\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 279.12it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48198104739362263\n",
      "[test] Raw Hits@1: 35.72\n",
      "[test] Raw Hits@3: 55.400000000000006\n",
      "[test] Raw Hits@5: 63.3\n",
      "[test] Raw Hits@10: 72.26\n",
      "[test] Filtered MRR: 0.640965759839285\n",
      "[test] Filtered Hits@1: 54.50000000000001\n",
      "[test] Filtered Hits@3: 70.86\n",
      "[test] Filtered Hits@5: 75.7\n",
      "[test] Filtered Hits@10: 81.26\n",
      "INFO:tensorflow:Epoch 72\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-66953\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 66954 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 51.989017, step = 66954\n",
      "INFO:tensorflow:global_step/sec: 1.91007\n",
      "INFO:tensorflow:loss = 61.584927, step = 67054 (52.355 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03188\n",
      "INFO:tensorflow:loss = 53.751034, step = 67154 (49.217 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92726\n",
      "INFO:tensorflow:loss = 94.02687, step = 67254 (51.886 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91321\n",
      "INFO:tensorflow:loss = 57.134052, step = 67354 (52.268 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.74655\n",
      "INFO:tensorflow:loss = 65.99816, step = 67454 (57.256 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.78512\n",
      "INFO:tensorflow:loss = 49.204517, step = 67554 (56.020 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97618\n",
      "INFO:tensorflow:loss = 54.10393, step = 67654 (50.602 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96615\n",
      "INFO:tensorflow:loss = 64.703354, step = 67754 (50.860 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99871\n",
      "INFO:tensorflow:loss = 56.946873, step = 67854 (50.032 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 67896 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 87.5946.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-67896\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 284.41it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:51<00:00, 44.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48746969175840643\n",
      "[test] Raw Hits@1: 36.4\n",
      "[test] Raw Hits@3: 56.06\n",
      "[test] Raw Hits@5: 63.54\n",
      "[test] Raw Hits@10: 72.32\n",
      "[test] Filtered MRR: 0.6476238795094202\n",
      "[test] Filtered Hits@1: 55.379999999999995\n",
      "[test] Filtered Hits@3: 71.08\n",
      "[test] Filtered Hits@5: 75.94\n",
      "[test] Filtered Hits@10: 80.82000000000001\n",
      "INFO:tensorflow:Epoch 73\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-67896\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 67897 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 67.20226, step = 67897\n",
      "INFO:tensorflow:global_step/sec: 1.94057\n",
      "INFO:tensorflow:loss = 60.082394, step = 67997 (51.532 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.87895\n",
      "INFO:tensorflow:loss = 64.955925, step = 68097 (53.221 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.81398\n",
      "INFO:tensorflow:loss = 83.65957, step = 68197 (55.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92612\n",
      "INFO:tensorflow:loss = 58.437897, step = 68297 (51.918 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.89935\n",
      "INFO:tensorflow:loss = 75.467255, step = 68397 (52.649 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91016\n",
      "INFO:tensorflow:loss = 53.512863, step = 68497 (52.352 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.01223\n",
      "INFO:tensorflow:loss = 55.92442, step = 68597 (49.696 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.0339\n",
      "INFO:tensorflow:loss = 72.75484, step = 68697 (49.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05675\n",
      "INFO:tensorflow:loss = 57.221928, step = 68797 (48.621 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 68839 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 41.721092.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-68839\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 296.38it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:44<00:00, 47.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4859917404645486\n",
      "[test] Raw Hits@1: 36.14\n",
      "[test] Raw Hits@3: 55.67999999999999\n",
      "[test] Raw Hits@5: 63.82\n",
      "[test] Raw Hits@10: 72.82\n",
      "[test] Filtered MRR: 0.651482764680926\n",
      "[test] Filtered Hits@1: 55.300000000000004\n",
      "[test] Filtered Hits@3: 72.39999999999999\n",
      "[test] Filtered Hits@5: 76.74\n",
      "[test] Filtered Hits@10: 82.02000000000001\n",
      "INFO:tensorflow:Epoch 74\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-68839\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 68840 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 51.203712, step = 68840\n",
      "INFO:tensorflow:global_step/sec: 2.00474\n",
      "INFO:tensorflow:loss = 49.041332, step = 68940 (49.883 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.16806\n",
      "INFO:tensorflow:loss = 71.329254, step = 69040 (46.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20596\n",
      "INFO:tensorflow:loss = 51.126522, step = 69140 (45.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19867\n",
      "INFO:tensorflow:loss = 54.875347, step = 69240 (45.482 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20707\n",
      "INFO:tensorflow:loss = 62.1419, step = 69340 (45.309 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.199\n",
      "INFO:tensorflow:loss = 70.37517, step = 69440 (45.476 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19921\n",
      "INFO:tensorflow:loss = 63.809128, step = 69540 (45.470 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19865\n",
      "INFO:tensorflow:loss = 57.290592, step = 69640 (45.482 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20123\n",
      "INFO:tensorflow:loss = 49.397263, step = 69740 (45.429 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 69782 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 75.24796.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-69782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.86it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4823800221526115\n",
      "[test] Raw Hits@1: 36.16\n",
      "[test] Raw Hits@3: 55.26\n",
      "[test] Raw Hits@5: 62.739999999999995\n",
      "[test] Raw Hits@10: 71.54\n",
      "[test] Filtered MRR: 0.6325213584330848\n",
      "[test] Filtered Hits@1: 53.2\n",
      "[test] Filtered Hits@3: 69.94\n",
      "[test] Filtered Hits@5: 74.9\n",
      "[test] Filtered Hits@10: 80.9\n",
      "INFO:tensorflow:Epoch 75\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-69782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 69783 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 64.518364, step = 69783\n",
      "INFO:tensorflow:global_step/sec: 2.18973\n",
      "INFO:tensorflow:loss = 35.124695, step = 69883 (45.669 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19654\n",
      "INFO:tensorflow:loss = 55.280968, step = 69983 (45.526 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1993\n",
      "INFO:tensorflow:loss = 70.61378, step = 70083 (45.469 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20407\n",
      "INFO:tensorflow:loss = 66.91145, step = 70183 (45.370 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20122\n",
      "INFO:tensorflow:loss = 48.02274, step = 70283 (45.429 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1979\n",
      "INFO:tensorflow:loss = 33.553745, step = 70383 (45.498 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20448\n",
      "INFO:tensorflow:loss = 72.65862, step = 70483 (45.362 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19772\n",
      "INFO:tensorflow:loss = 68.29918, step = 70583 (45.502 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20322\n",
      "INFO:tensorflow:loss = 77.87528, step = 70683 (45.388 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 70725 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 54.390343.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-70725\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.60it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48464950059279416\n",
      "[test] Raw Hits@1: 36.059999999999995\n",
      "[test] Raw Hits@3: 55.54\n",
      "[test] Raw Hits@5: 63.68000000000001\n",
      "[test] Raw Hits@10: 72.28\n",
      "[test] Filtered MRR: 0.6457966873405335\n",
      "[test] Filtered Hits@1: 54.98\n",
      "[test] Filtered Hits@3: 71.14\n",
      "[test] Filtered Hits@5: 76.12\n",
      "[test] Filtered Hits@10: 81.32000000000001\n",
      "INFO:tensorflow:Epoch 76\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-70725\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 70726 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 50.953182, step = 70726\n",
      "INFO:tensorflow:global_step/sec: 2.19123\n",
      "INFO:tensorflow:loss = 46.257717, step = 70826 (45.638 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20584\n",
      "INFO:tensorflow:loss = 73.7, step = 70926 (45.334 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20335\n",
      "INFO:tensorflow:loss = 40.55151, step = 71026 (45.386 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20395\n",
      "INFO:tensorflow:loss = 32.111317, step = 71126 (45.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20315\n",
      "INFO:tensorflow:loss = 54.68592, step = 71226 (45.390 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20335\n",
      "INFO:tensorflow:loss = 79.435196, step = 71326 (45.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20374\n",
      "INFO:tensorflow:loss = 90.97357, step = 71426 (45.377 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20413\n",
      "INFO:tensorflow:loss = 83.84475, step = 71526 (45.369 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19816\n",
      "INFO:tensorflow:loss = 72.49722, step = 71626 (45.493 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 71668 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 53.157337.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-71668\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.24it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48548492961007844\n",
      "[test] Raw Hits@1: 36.04\n",
      "[test] Raw Hits@3: 55.779999999999994\n",
      "[test] Raw Hits@5: 63.68000000000001\n",
      "[test] Raw Hits@10: 72.28\n",
      "[test] Filtered MRR: 0.6458443351465114\n",
      "[test] Filtered Hits@1: 54.86\n",
      "[test] Filtered Hits@3: 71.12\n",
      "[test] Filtered Hits@5: 76.18\n",
      "[test] Filtered Hits@10: 81.82000000000001\n",
      "INFO:tensorflow:Epoch 77\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-71668\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 71669 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 65.33071, step = 71669\n",
      "INFO:tensorflow:global_step/sec: 2.1751\n",
      "INFO:tensorflow:loss = 53.346447, step = 71769 (45.976 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.16987\n",
      "INFO:tensorflow:loss = 46.238293, step = 71869 (46.086 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18124\n",
      "INFO:tensorflow:loss = 47.698193, step = 71969 (45.846 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17692\n",
      "INFO:tensorflow:loss = 59.807526, step = 72069 (45.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17871\n",
      "INFO:tensorflow:loss = 90.17487, step = 72169 (45.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17574\n",
      "INFO:tensorflow:loss = 75.1819, step = 72269 (45.961 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17981\n",
      "INFO:tensorflow:loss = 51.738056, step = 72369 (45.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18429\n",
      "INFO:tensorflow:loss = 64.2804, step = 72469 (45.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17876\n",
      "INFO:tensorflow:loss = 98.36806, step = 72569 (45.898 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 72611 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 71.33298.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-72611\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.59it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48594891508675353\n",
      "[test] Raw Hits@1: 36.08\n",
      "[test] Raw Hits@3: 56.14\n",
      "[test] Raw Hits@5: 63.9\n",
      "[test] Raw Hits@10: 72.34\n",
      "[test] Filtered MRR: 0.6490581269752872\n",
      "[test] Filtered Hits@1: 55.32\n",
      "[test] Filtered Hits@3: 71.5\n",
      "[test] Filtered Hits@5: 76.3\n",
      "[test] Filtered Hits@10: 81.44\n",
      "INFO:tensorflow:Epoch 78\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-72611\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 72612 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 47.187107, step = 72612\n",
      "INFO:tensorflow:global_step/sec: 2.16341\n",
      "INFO:tensorflow:loss = 50.156956, step = 72712 (46.224 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17611\n",
      "INFO:tensorflow:loss = 55.5583, step = 72812 (45.954 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17737\n",
      "INFO:tensorflow:loss = 52.325752, step = 72912 (45.927 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17927\n",
      "INFO:tensorflow:loss = 63.90041, step = 73012 (45.887 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17782\n",
      "INFO:tensorflow:loss = 28.873058, step = 73112 (45.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17826\n",
      "INFO:tensorflow:loss = 40.656967, step = 73212 (45.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17268\n",
      "INFO:tensorflow:loss = 63.31466, step = 73312 (46.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17717\n",
      "INFO:tensorflow:loss = 61.24167, step = 73412 (45.931 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17657\n",
      "INFO:tensorflow:loss = 52.974934, step = 73512 (45.944 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 73554 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 67.490555.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-73554\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.86it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4877869966492324\n",
      "[test] Raw Hits@1: 36.52\n",
      "[test] Raw Hits@3: 55.82\n",
      "[test] Raw Hits@5: 63.4\n",
      "[test] Raw Hits@10: 72.36\n",
      "[test] Filtered MRR: 0.6483641099410205\n",
      "[test] Filtered Hits@1: 55.24\n",
      "[test] Filtered Hits@3: 71.24000000000001\n",
      "[test] Filtered Hits@5: 76.02\n",
      "[test] Filtered Hits@10: 81.66\n",
      "INFO:tensorflow:Epoch 79\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-73554\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 73555 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 53.811325, step = 73555\n",
      "INFO:tensorflow:global_step/sec: 2.16035\n",
      "INFO:tensorflow:loss = 62.65503, step = 73655 (46.290 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17653\n",
      "INFO:tensorflow:loss = 56.2028, step = 73755 (45.945 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18049\n",
      "INFO:tensorflow:loss = 36.163193, step = 73855 (45.861 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17915\n",
      "INFO:tensorflow:loss = 60.73207, step = 73955 (45.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17482\n",
      "INFO:tensorflow:loss = 48.74621, step = 74055 (45.981 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17988\n",
      "INFO:tensorflow:loss = 55.78957, step = 74155 (45.874 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18105\n",
      "INFO:tensorflow:loss = 67.74438, step = 74255 (45.849 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17708\n",
      "INFO:tensorflow:loss = 78.04348, step = 74355 (45.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17764\n",
      "INFO:tensorflow:loss = 51.83852, step = 74455 (45.921 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 74497 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 53.20468.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-74497\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.87it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48753489810930695\n",
      "[test] Raw Hits@1: 36.54\n",
      "[test] Raw Hits@3: 55.94\n",
      "[test] Raw Hits@5: 63.63999999999999\n",
      "[test] Raw Hits@10: 72.06\n",
      "[test] Filtered MRR: 0.6480283721778788\n",
      "[test] Filtered Hits@1: 55.120000000000005\n",
      "[test] Filtered Hits@3: 71.48\n",
      "[test] Filtered Hits@5: 76.24\n",
      "[test] Filtered Hits@10: 81.8\n",
      "INFO:tensorflow:Epoch 80\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-74497\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 74498 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 73.097694, step = 74498\n",
      "INFO:tensorflow:global_step/sec: 2.19062\n",
      "INFO:tensorflow:loss = 87.29516, step = 74598 (45.650 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2044\n",
      "INFO:tensorflow:loss = 40.5222, step = 74698 (45.364 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20033\n",
      "INFO:tensorflow:loss = 59.088413, step = 74798 (45.448 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2075\n",
      "INFO:tensorflow:loss = 46.249302, step = 74898 (45.300 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20602\n",
      "INFO:tensorflow:loss = 59.477337, step = 74998 (45.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20144\n",
      "INFO:tensorflow:loss = 27.629652, step = 75098 (45.425 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20412\n",
      "INFO:tensorflow:loss = 42.98732, step = 75198 (45.369 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20478\n",
      "INFO:tensorflow:loss = 47.026196, step = 75298 (45.356 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20637\n",
      "INFO:tensorflow:loss = 56.06674, step = 75398 (45.323 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 75440 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 56.487335.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-75440\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.66it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4881839333029362\n",
      "[test] Raw Hits@1: 36.44\n",
      "[test] Raw Hits@3: 55.94\n",
      "[test] Raw Hits@5: 63.88\n",
      "[test] Raw Hits@10: 72.38\n",
      "[test] Filtered MRR: 0.6485716592653774\n",
      "[test] Filtered Hits@1: 55.059999999999995\n",
      "[test] Filtered Hits@3: 71.3\n",
      "[test] Filtered Hits@5: 76.74\n",
      "[test] Filtered Hits@10: 81.98\n",
      "INFO:tensorflow:Epoch 81\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-75440\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 75441 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 34.47193, step = 75441\n",
      "INFO:tensorflow:global_step/sec: 2.16506\n",
      "INFO:tensorflow:loss = 21.931416, step = 75541 (46.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17841\n",
      "INFO:tensorflow:loss = 58.66832, step = 75641 (45.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17791\n",
      "INFO:tensorflow:loss = 66.20342, step = 75741 (45.916 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18123\n",
      "INFO:tensorflow:loss = 50.208836, step = 75841 (45.846 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17981\n",
      "INFO:tensorflow:loss = 72.504234, step = 75941 (45.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18403\n",
      "INFO:tensorflow:loss = 30.840239, step = 76041 (45.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17776\n",
      "INFO:tensorflow:loss = 43.873177, step = 76141 (45.918 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18024\n",
      "INFO:tensorflow:loss = 66.02168, step = 76241 (45.867 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17899\n",
      "INFO:tensorflow:loss = 68.12059, step = 76341 (45.893 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 76383 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 45.57391.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-76383\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.37it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.48928256012074756\n",
      "[test] Raw Hits@1: 36.44\n",
      "[test] Raw Hits@3: 56.279999999999994\n",
      "[test] Raw Hits@5: 63.94\n",
      "[test] Raw Hits@10: 72.96000000000001\n",
      "[test] Filtered MRR: 0.6482130421142752\n",
      "[test] Filtered Hits@1: 55.24\n",
      "[test] Filtered Hits@3: 71.17999999999999\n",
      "[test] Filtered Hits@5: 76.22\n",
      "[test] Filtered Hits@10: 81.36\n",
      "INFO:tensorflow:Epoch 82\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-76383\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 76384 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 67.48275, step = 76384\n",
      "INFO:tensorflow:global_step/sec: 2.16673\n",
      "INFO:tensorflow:loss = 40.735504, step = 76484 (46.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.14536\n",
      "INFO:tensorflow:loss = 53.545143, step = 76584 (46.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18445\n",
      "INFO:tensorflow:loss = 40.688293, step = 76684 (45.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1842\n",
      "INFO:tensorflow:loss = 43.083397, step = 76784 (45.783 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18723\n",
      "INFO:tensorflow:loss = 56.4765, step = 76884 (45.720 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18365\n",
      "INFO:tensorflow:loss = 48.220566, step = 76984 (45.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18655\n",
      "INFO:tensorflow:loss = 74.57652, step = 77084 (45.734 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18015\n",
      "INFO:tensorflow:loss = 65.60009, step = 77184 (45.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17986\n",
      "INFO:tensorflow:loss = 34.985542, step = 77284 (45.875 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 77326 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 51.118195.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-77326\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.57it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4937993203185689\n",
      "[test] Raw Hits@1: 37.36\n",
      "[test] Raw Hits@3: 56.279999999999994\n",
      "[test] Raw Hits@5: 63.800000000000004\n",
      "[test] Raw Hits@10: 72.08\n",
      "[test] Filtered MRR: 0.6520712357299796\n",
      "[test] Filtered Hits@1: 56.16\n",
      "[test] Filtered Hits@3: 71.24000000000001\n",
      "[test] Filtered Hits@5: 75.56\n",
      "[test] Filtered Hits@10: 80.74\n",
      "INFO:tensorflow:Epoch 83\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-77326\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 77327 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 103.345856, step = 77327\n",
      "INFO:tensorflow:global_step/sec: 2.16338\n",
      "INFO:tensorflow:loss = 42.476627, step = 77427 (46.225 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17533\n",
      "INFO:tensorflow:loss = 33.58263, step = 77527 (45.970 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17136\n",
      "INFO:tensorflow:loss = 59.608257, step = 77627 (46.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18416\n",
      "INFO:tensorflow:loss = 44.24533, step = 77727 (45.784 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18682\n",
      "INFO:tensorflow:loss = 34.28272, step = 77827 (45.729 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18049\n",
      "INFO:tensorflow:loss = 59.268383, step = 77927 (45.861 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17976\n",
      "INFO:tensorflow:loss = 36.563965, step = 78027 (45.876 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18802\n",
      "INFO:tensorflow:loss = 51.938732, step = 78127 (45.703 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18182\n",
      "INFO:tensorflow:loss = 69.054474, step = 78227 (45.833 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 78269 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.02746.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-78269\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.45it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4914305648145713\n",
      "[test] Raw Hits@1: 36.66\n",
      "[test] Raw Hits@3: 56.68\n",
      "[test] Raw Hits@5: 63.94\n",
      "[test] Raw Hits@10: 72.76\n",
      "[test] Filtered MRR: 0.6529214143577605\n",
      "[test] Filtered Hits@1: 55.7\n",
      "[test] Filtered Hits@3: 71.88\n",
      "[test] Filtered Hits@5: 76.5\n",
      "[test] Filtered Hits@10: 82.0\n",
      "INFO:tensorflow:Epoch 84\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-78269\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 78270 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 69.99813, step = 78270\n",
      "INFO:tensorflow:global_step/sec: 2.18877\n",
      "INFO:tensorflow:loss = 49.700245, step = 78370 (45.689 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19969\n",
      "INFO:tensorflow:loss = 44.98837, step = 78470 (45.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20007\n",
      "INFO:tensorflow:loss = 41.560684, step = 78570 (45.453 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20057\n",
      "INFO:tensorflow:loss = 90.136856, step = 78670 (45.443 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19129\n",
      "INFO:tensorflow:loss = 33.784546, step = 78770 (45.635 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20605\n",
      "INFO:tensorflow:loss = 53.125233, step = 78870 (45.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20149\n",
      "INFO:tensorflow:loss = 72.4976, step = 78970 (45.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20398\n",
      "INFO:tensorflow:loss = 73.32664, step = 79070 (45.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2119\n",
      "INFO:tensorflow:loss = 60.689808, step = 79170 (45.210 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 79212 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 93.346695.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-79212\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.30it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4909929193082713\n",
      "[test] Raw Hits@1: 36.6\n",
      "[test] Raw Hits@3: 56.74\n",
      "[test] Raw Hits@5: 64.18\n",
      "[test] Raw Hits@10: 72.11999999999999\n",
      "[test] Filtered MRR: 0.6521187307868764\n",
      "[test] Filtered Hits@1: 55.82\n",
      "[test] Filtered Hits@3: 71.92\n",
      "[test] Filtered Hits@5: 75.98\n",
      "[test] Filtered Hits@10: 81.14\n",
      "INFO:tensorflow:Epoch 85\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-79212\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 79213 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 79.18764, step = 79213\n",
      "INFO:tensorflow:global_step/sec: 2.1957\n",
      "INFO:tensorflow:loss = 31.624958, step = 79313 (45.545 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1974\n",
      "INFO:tensorflow:loss = 50.42149, step = 79413 (45.508 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20554\n",
      "INFO:tensorflow:loss = 43.902756, step = 79513 (45.341 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19842\n",
      "INFO:tensorflow:loss = 73.83425, step = 79613 (45.487 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20797\n",
      "INFO:tensorflow:loss = 69.09742, step = 79713 (45.290 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2059\n",
      "INFO:tensorflow:loss = 69.65172, step = 79813 (45.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20301\n",
      "INFO:tensorflow:loss = 39.94377, step = 79913 (45.392 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20137\n",
      "INFO:tensorflow:loss = 63.57795, step = 80013 (45.426 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20375\n",
      "INFO:tensorflow:loss = 58.918884, step = 80113 (45.377 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 80155 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 35.32635.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-80155\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 305.06it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49156400189868454\n",
      "[test] Raw Hits@1: 36.620000000000005\n",
      "[test] Raw Hits@3: 56.98\n",
      "[test] Raw Hits@5: 64.08\n",
      "[test] Raw Hits@10: 72.84\n",
      "[test] Filtered MRR: 0.6558066288900746\n",
      "[test] Filtered Hits@1: 55.82\n",
      "[test] Filtered Hits@3: 72.39999999999999\n",
      "[test] Filtered Hits@5: 76.9\n",
      "[test] Filtered Hits@10: 82.44\n",
      "INFO:tensorflow:Epoch 86\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-80155\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 80156 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 81.274826, step = 80156\n",
      "INFO:tensorflow:global_step/sec: 2.18567\n",
      "INFO:tensorflow:loss = 46.32774, step = 80256 (45.754 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20256\n",
      "INFO:tensorflow:loss = 59.366417, step = 80356 (45.402 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19882\n",
      "INFO:tensorflow:loss = 60.906967, step = 80456 (45.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20268\n",
      "INFO:tensorflow:loss = 70.668205, step = 80556 (45.399 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2021\n",
      "INFO:tensorflow:loss = 40.20686, step = 80656 (45.411 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19987\n",
      "INFO:tensorflow:loss = 34.34185, step = 80756 (45.457 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19697\n",
      "INFO:tensorflow:loss = 62.07367, step = 80856 (45.517 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20045\n",
      "INFO:tensorflow:loss = 37.954594, step = 80956 (45.445 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2043\n",
      "INFO:tensorflow:loss = 83.0067, step = 81056 (45.366 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 81098 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 32.98683.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-81098\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.40it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49204711916655836\n",
      "[test] Raw Hits@1: 36.919999999999995\n",
      "[test] Raw Hits@3: 56.36\n",
      "[test] Raw Hits@5: 64.25999999999999\n",
      "[test] Raw Hits@10: 72.68\n",
      "[test] Filtered MRR: 0.6530332772563282\n",
      "[test] Filtered Hits@1: 55.74\n",
      "[test] Filtered Hits@3: 71.6\n",
      "[test] Filtered Hits@5: 76.68\n",
      "[test] Filtered Hits@10: 82.17999999999999\n",
      "INFO:tensorflow:Epoch 87\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-81098\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 81099 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 47.49277, step = 81099\n",
      "INFO:tensorflow:global_step/sec: 2.19044\n",
      "INFO:tensorflow:loss = 59.729362, step = 81199 (45.654 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20017\n",
      "INFO:tensorflow:loss = 44.438877, step = 81299 (45.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20978\n",
      "INFO:tensorflow:loss = 48.721363, step = 81399 (45.254 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20629\n",
      "INFO:tensorflow:loss = 51.910793, step = 81499 (45.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20195\n",
      "INFO:tensorflow:loss = 87.23783, step = 81599 (45.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2031\n",
      "INFO:tensorflow:loss = 41.13552, step = 81699 (45.391 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.21293\n",
      "INFO:tensorflow:loss = 69.81024, step = 81799 (45.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2057\n",
      "INFO:tensorflow:loss = 54.47237, step = 81899 (45.337 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20907\n",
      "INFO:tensorflow:loss = 41.843784, step = 81999 (45.268 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 82041 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 57.38127.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-82041\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.25it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49251626277521837\n",
      "[test] Raw Hits@1: 36.88\n",
      "[test] Raw Hits@3: 56.46\n",
      "[test] Raw Hits@5: 64.36\n",
      "[test] Raw Hits@10: 73.08\n",
      "[test] Filtered MRR: 0.6590580958004434\n",
      "[test] Filtered Hits@1: 56.3\n",
      "[test] Filtered Hits@3: 72.38\n",
      "[test] Filtered Hits@5: 77.38000000000001\n",
      "[test] Filtered Hits@10: 82.62\n",
      "INFO:tensorflow:Epoch 88\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-82041\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 82042 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 49.943264, step = 82042\n",
      "INFO:tensorflow:global_step/sec: 2.18866\n",
      "INFO:tensorflow:loss = 76.0819, step = 82142 (45.691 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19597\n",
      "INFO:tensorflow:loss = 34.951958, step = 82242 (45.538 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19095\n",
      "INFO:tensorflow:loss = 52.31087, step = 82342 (45.643 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20603\n",
      "INFO:tensorflow:loss = 44.82361, step = 82442 (45.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20506\n",
      "INFO:tensorflow:loss = 47.51698, step = 82542 (45.350 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20001\n",
      "INFO:tensorflow:loss = 105.96801, step = 82642 (45.454 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20337\n",
      "INFO:tensorflow:loss = 71.817764, step = 82742 (45.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20578\n",
      "INFO:tensorflow:loss = 65.24822, step = 82842 (45.335 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20235\n",
      "INFO:tensorflow:loss = 63.645206, step = 82942 (45.406 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 82984 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 63.212.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-82984\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.49it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4938558503118649\n",
      "[test] Raw Hits@1: 37.16\n",
      "[test] Raw Hits@3: 56.48\n",
      "[test] Raw Hits@5: 64.42\n",
      "[test] Raw Hits@10: 73.1\n",
      "[test] Filtered MRR: 0.6585374514498042\n",
      "[test] Filtered Hits@1: 56.56\n",
      "[test] Filtered Hits@3: 72.26\n",
      "[test] Filtered Hits@5: 76.82\n",
      "[test] Filtered Hits@10: 82.42\n",
      "INFO:tensorflow:Epoch 89\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-82984\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 82985 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 47.449818, step = 82985\n",
      "INFO:tensorflow:global_step/sec: 1.89745\n",
      "INFO:tensorflow:loss = 66.91585, step = 83085 (52.704 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.91933\n",
      "INFO:tensorflow:loss = 49.593117, step = 83185 (52.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97302\n",
      "INFO:tensorflow:loss = 83.80297, step = 83285 (50.684 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07491\n",
      "INFO:tensorflow:loss = 32.5874, step = 83385 (48.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.12903\n",
      "INFO:tensorflow:loss = 33.9263, step = 83485 (46.970 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.08412\n",
      "INFO:tensorflow:loss = 64.910835, step = 83585 (47.982 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97816\n",
      "INFO:tensorflow:loss = 49.1944, step = 83685 (50.552 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.08543\n",
      "INFO:tensorflow:loss = 60.954494, step = 83785 (47.951 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03344\n",
      "INFO:tensorflow:loss = 43.565956, step = 83885 (49.178 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 83927 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 22.280947.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-83927\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 294.32it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:47<00:00, 46.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49609678006704816\n",
      "[test] Raw Hits@1: 37.46\n",
      "[test] Raw Hits@3: 56.699999999999996\n",
      "[test] Raw Hits@5: 64.5\n",
      "[test] Raw Hits@10: 73.2\n",
      "[test] Filtered MRR: 0.6626737370955075\n",
      "[test] Filtered Hits@1: 57.220000000000006\n",
      "[test] Filtered Hits@3: 72.42\n",
      "[test] Filtered Hits@5: 77.08\n",
      "[test] Filtered Hits@10: 82.44\n",
      "INFO:tensorflow:Epoch 90\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-83927\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 83928 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 38.041134, step = 83928\n",
      "INFO:tensorflow:global_step/sec: 1.99315\n",
      "INFO:tensorflow:loss = 56.41884, step = 84028 (50.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.96087\n",
      "INFO:tensorflow:loss = 64.14218, step = 84128 (50.997 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.03085\n",
      "INFO:tensorflow:loss = 47.50872, step = 84228 (49.240 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.13848\n",
      "INFO:tensorflow:loss = 41.656143, step = 84328 (46.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04488\n",
      "INFO:tensorflow:loss = 43.125664, step = 84428 (48.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04753\n",
      "INFO:tensorflow:loss = 52.877785, step = 84528 (48.840 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.88472\n",
      "INFO:tensorflow:loss = 43.280457, step = 84628 (53.057 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.92894\n",
      "INFO:tensorflow:loss = 53.39264, step = 84728 (51.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.98264\n",
      "INFO:tensorflow:loss = 71.923355, step = 84828 (50.438 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 84870 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 55.36712.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-84870\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 299.27it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:46<00:00, 46.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.494527727510121\n",
      "[test] Raw Hits@1: 37.059999999999995\n",
      "[test] Raw Hits@3: 56.8\n",
      "[test] Raw Hits@5: 64.88000000000001\n",
      "[test] Raw Hits@10: 73.38\n",
      "[test] Filtered MRR: 0.6629262668667185\n",
      "[test] Filtered Hits@1: 56.879999999999995\n",
      "[test] Filtered Hits@3: 72.86\n",
      "[test] Filtered Hits@5: 77.62\n",
      "[test] Filtered Hits@10: 82.6\n",
      "INFO:tensorflow:Epoch 91\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-84870\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 84871 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 37.442184, step = 84871\n",
      "INFO:tensorflow:global_step/sec: 2.0216\n",
      "INFO:tensorflow:loss = 65.93133, step = 84971 (49.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.04393\n",
      "INFO:tensorflow:loss = 39.425858, step = 85071 (48.925 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.93942\n",
      "INFO:tensorflow:loss = 49.210144, step = 85171 (51.562 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.90796\n",
      "INFO:tensorflow:loss = 40.542393, step = 85271 (52.412 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00452\n",
      "INFO:tensorflow:loss = 59.98381, step = 85371 (49.888 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.00005\n",
      "INFO:tensorflow:loss = 37.830112, step = 85471 (49.998 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.07541\n",
      "INFO:tensorflow:loss = 62.501545, step = 85571 (48.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99522\n",
      "INFO:tensorflow:loss = 87.42454, step = 85671 (50.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.97531\n",
      "INFO:tensorflow:loss = 46.234756, step = 85771 (50.625 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 85813 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 58.61228.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-85813\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:17<00:00, 286.33it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:49<00:00, 45.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49321814156514093\n",
      "[test] Raw Hits@1: 36.8\n",
      "[test] Raw Hits@3: 56.49999999999999\n",
      "[test] Raw Hits@5: 64.60000000000001\n",
      "[test] Raw Hits@10: 73.26\n",
      "[test] Filtered MRR: 0.661882596080468\n",
      "[test] Filtered Hits@1: 56.82000000000001\n",
      "[test] Filtered Hits@3: 72.34\n",
      "[test] Filtered Hits@5: 77.52\n",
      "[test] Filtered Hits@10: 82.66\n",
      "INFO:tensorflow:Epoch 92\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-85813\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 85814 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 46.697914, step = 85814\n",
      "INFO:tensorflow:global_step/sec: 1.94699\n",
      "INFO:tensorflow:loss = 51.47531, step = 85914 (51.362 sec)\n",
      "INFO:tensorflow:global_step/sec: 1.99479\n",
      "INFO:tensorflow:loss = 29.49902, step = 86014 (50.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.05838\n",
      "INFO:tensorflow:loss = 30.983746, step = 86114 (48.582 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19068\n",
      "INFO:tensorflow:loss = 55.016422, step = 86214 (45.648 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18484\n",
      "INFO:tensorflow:loss = 54.545357, step = 86314 (45.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18541\n",
      "INFO:tensorflow:loss = 44.53463, step = 86414 (45.758 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18182\n",
      "INFO:tensorflow:loss = 60.024826, step = 86514 (45.833 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18347\n",
      "INFO:tensorflow:loss = 85.44406, step = 86614 (45.799 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19176\n",
      "INFO:tensorflow:loss = 24.280916, step = 86714 (45.625 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 86756 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 38.0479.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-86756\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.98it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4972272561371795\n",
      "[test] Raw Hits@1: 37.28\n",
      "[test] Raw Hits@3: 57.08\n",
      "[test] Raw Hits@5: 65.06\n",
      "[test] Raw Hits@10: 73.46000000000001\n",
      "[test] Filtered MRR: 0.6661980704778282\n",
      "[test] Filtered Hits@1: 57.49999999999999\n",
      "[test] Filtered Hits@3: 73.04\n",
      "[test] Filtered Hits@5: 78.02\n",
      "[test] Filtered Hits@10: 82.74000000000001\n",
      "INFO:tensorflow:Epoch 93\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-86756\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 86757 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 40.76749, step = 86757\n",
      "INFO:tensorflow:global_step/sec: 2.1916\n",
      "INFO:tensorflow:loss = 43.465603, step = 86857 (45.630 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20017\n",
      "INFO:tensorflow:loss = 42.763996, step = 86957 (45.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20549\n",
      "INFO:tensorflow:loss = 48.71015, step = 87057 (45.342 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20839\n",
      "INFO:tensorflow:loss = 50.70357, step = 87157 (45.281 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20519\n",
      "INFO:tensorflow:loss = 76.00604, step = 87257 (45.347 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20762\n",
      "INFO:tensorflow:loss = 52.32368, step = 87357 (45.298 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20554\n",
      "INFO:tensorflow:loss = 55.283962, step = 87457 (45.340 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20783\n",
      "INFO:tensorflow:loss = 45.79098, step = 87557 (45.293 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.21169\n",
      "INFO:tensorflow:loss = 52.798264, step = 87657 (45.214 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 87699 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 36.26794.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-87699\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.32it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49473642239156224\n",
      "[test] Raw Hits@1: 37.059999999999995\n",
      "[test] Raw Hits@3: 56.720000000000006\n",
      "[test] Raw Hits@5: 64.36\n",
      "[test] Raw Hits@10: 73.11999999999999\n",
      "[test] Filtered MRR: 0.6582972397413419\n",
      "[test] Filtered Hits@1: 56.46\n",
      "[test] Filtered Hits@3: 72.04\n",
      "[test] Filtered Hits@5: 77.08\n",
      "[test] Filtered Hits@10: 82.32000000000001\n",
      "INFO:tensorflow:Epoch 94\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-87699\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 87700 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 34.379627, step = 87700\n",
      "INFO:tensorflow:global_step/sec: 2.1916\n",
      "INFO:tensorflow:loss = 45.82006, step = 87800 (45.630 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20635\n",
      "INFO:tensorflow:loss = 33.437447, step = 87900 (45.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20193\n",
      "INFO:tensorflow:loss = 58.639507, step = 88000 (45.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20393\n",
      "INFO:tensorflow:loss = 50.86207, step = 88100 (45.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20033\n",
      "INFO:tensorflow:loss = 38.40488, step = 88200 (45.448 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.21001\n",
      "INFO:tensorflow:loss = 29.683416, step = 88300 (45.249 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20642\n",
      "INFO:tensorflow:loss = 62.306362, step = 88400 (45.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20442\n",
      "INFO:tensorflow:loss = 34.540607, step = 88500 (45.363 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20014\n",
      "INFO:tensorflow:loss = 59.690403, step = 88600 (45.452 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 88642 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 33.234825.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-88642\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.37it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4982442989318542\n",
      "[test] Raw Hits@1: 37.519999999999996\n",
      "[test] Raw Hits@3: 57.06\n",
      "[test] Raw Hits@5: 65.10000000000001\n",
      "[test] Raw Hits@10: 73.2\n",
      "[test] Filtered MRR: 0.6647438139119819\n",
      "[test] Filtered Hits@1: 57.24\n",
      "[test] Filtered Hits@3: 72.78\n",
      "[test] Filtered Hits@5: 77.66\n",
      "[test] Filtered Hits@10: 82.36\n",
      "INFO:tensorflow:Epoch 95\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-88642\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 88643 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 46.506775, step = 88643\n",
      "INFO:tensorflow:global_step/sec: 2.17079\n",
      "INFO:tensorflow:loss = 41.784786, step = 88743 (46.067 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17648\n",
      "INFO:tensorflow:loss = 34.5433, step = 88843 (45.946 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1874\n",
      "INFO:tensorflow:loss = 43.5693, step = 88943 (45.716 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18161\n",
      "INFO:tensorflow:loss = 49.177486, step = 89043 (45.838 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18492\n",
      "INFO:tensorflow:loss = 52.830627, step = 89143 (45.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18732\n",
      "INFO:tensorflow:loss = 68.34547, step = 89243 (45.718 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.17859\n",
      "INFO:tensorflow:loss = 58.275475, step = 89343 (45.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19053\n",
      "INFO:tensorflow:loss = 57.65724, step = 89443 (45.651 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18489\n",
      "INFO:tensorflow:loss = 65.897606, step = 89543 (45.769 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 89585 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 43.578247.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-89585\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.37it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.4926903922101686\n",
      "[test] Raw Hits@1: 36.66\n",
      "[test] Raw Hits@3: 56.699999999999996\n",
      "[test] Raw Hits@5: 64.24\n",
      "[test] Raw Hits@10: 73.11999999999999\n",
      "[test] Filtered MRR: 0.6592792461506778\n",
      "[test] Filtered Hits@1: 56.279999999999994\n",
      "[test] Filtered Hits@3: 73.0\n",
      "[test] Filtered Hits@5: 77.64\n",
      "[test] Filtered Hits@10: 82.6\n",
      "INFO:tensorflow:Epoch 96\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-89585\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 89586 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 54.392643, step = 89586\n",
      "INFO:tensorflow:global_step/sec: 2.19523\n",
      "INFO:tensorflow:loss = 83.3854, step = 89686 (45.554 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19865\n",
      "INFO:tensorflow:loss = 62.49602, step = 89786 (45.482 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20832\n",
      "INFO:tensorflow:loss = 63.41483, step = 89886 (45.284 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20675\n",
      "INFO:tensorflow:loss = 44.4292, step = 89986 (45.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20385\n",
      "INFO:tensorflow:loss = 45.73439, step = 90086 (45.375 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20354\n",
      "INFO:tensorflow:loss = 54.495842, step = 90186 (45.382 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20834\n",
      "INFO:tensorflow:loss = 53.425568, step = 90286 (45.283 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20546\n",
      "INFO:tensorflow:loss = 45.05871, step = 90386 (45.342 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20636\n",
      "INFO:tensorflow:loss = 38.93901, step = 90486 (45.324 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 90528 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 39.10181.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-90528\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.47it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49826195991987715\n",
      "[test] Raw Hits@1: 37.64\n",
      "[test] Raw Hits@3: 56.68\n",
      "[test] Raw Hits@5: 64.68\n",
      "[test] Raw Hits@10: 72.88\n",
      "[test] Filtered MRR: 0.6674548648948608\n",
      "[test] Filtered Hits@1: 57.76\n",
      "[test] Filtered Hits@3: 72.78\n",
      "[test] Filtered Hits@5: 77.46\n",
      "[test] Filtered Hits@10: 82.48\n",
      "INFO:tensorflow:Epoch 97\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-90528\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 90529 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 47.32271, step = 90529\n",
      "INFO:tensorflow:global_step/sec: 2.18018\n",
      "INFO:tensorflow:loss = 51.06832, step = 90629 (45.869 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19942\n",
      "INFO:tensorflow:loss = 43.04228, step = 90729 (45.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20513\n",
      "INFO:tensorflow:loss = 54.28451, step = 90829 (45.349 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20103\n",
      "INFO:tensorflow:loss = 44.100006, step = 90929 (45.433 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20808\n",
      "INFO:tensorflow:loss = 60.24287, step = 91029 (45.288 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20693\n",
      "INFO:tensorflow:loss = 50.592693, step = 91129 (45.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2029\n",
      "INFO:tensorflow:loss = 60.95572, step = 91229 (45.395 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2041\n",
      "INFO:tensorflow:loss = 38.373306, step = 91329 (45.370 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20458\n",
      "INFO:tensorflow:loss = 42.069565, step = 91429 (45.360 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 91471 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 45.036015.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-91471\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.84it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49241576989891345\n",
      "[test] Raw Hits@1: 36.7\n",
      "[test] Raw Hits@3: 56.58\n",
      "[test] Raw Hits@5: 64.62\n",
      "[test] Raw Hits@10: 73.14\n",
      "[test] Filtered MRR: 0.6580206190156704\n",
      "[test] Filtered Hits@1: 56.34\n",
      "[test] Filtered Hits@3: 72.36\n",
      "[test] Filtered Hits@5: 77.48\n",
      "[test] Filtered Hits@10: 82.72\n",
      "INFO:tensorflow:Epoch 98\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-91471\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 91472 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 27.25086, step = 91472\n",
      "INFO:tensorflow:global_step/sec: 2.1688\n",
      "INFO:tensorflow:loss = 38.778038, step = 91572 (46.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18028\n",
      "INFO:tensorflow:loss = 39.82455, step = 91672 (45.866 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18823\n",
      "INFO:tensorflow:loss = 43.14599, step = 91772 (45.699 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18367\n",
      "INFO:tensorflow:loss = 116.196846, step = 91872 (45.794 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18987\n",
      "INFO:tensorflow:loss = 63.404182, step = 91972 (45.665 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19292\n",
      "INFO:tensorflow:loss = 44.896877, step = 92072 (45.602 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1867\n",
      "INFO:tensorflow:loss = 41.99992, step = 92172 (45.731 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.1831\n",
      "INFO:tensorflow:loss = 41.11662, step = 92272 (45.806 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19007\n",
      "INFO:tensorflow:loss = 46.89492, step = 92372 (45.661 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 92414 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 56.473984.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-92414\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 303.26it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49875417684733986\n",
      "[test] Raw Hits@1: 37.62\n",
      "[test] Raw Hits@3: 56.82000000000001\n",
      "[test] Raw Hits@5: 64.74\n",
      "[test] Raw Hits@10: 73.5\n",
      "[test] Filtered MRR: 0.671207172540884\n",
      "[test] Filtered Hits@1: 57.92\n",
      "[test] Filtered Hits@3: 73.3\n",
      "[test] Filtered Hits@5: 78.12\n",
      "[test] Filtered Hits@10: 83.3\n",
      "INFO:tensorflow:Epoch 99\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-92414\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 92415 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 77.26512, step = 92415\n",
      "INFO:tensorflow:global_step/sec: 2.18904\n",
      "INFO:tensorflow:loss = 30.501019, step = 92515 (45.683 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.19796\n",
      "INFO:tensorflow:loss = 49.29852, step = 92615 (45.497 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20563\n",
      "INFO:tensorflow:loss = 47.057243, step = 92715 (45.339 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20655\n",
      "INFO:tensorflow:loss = 58.66559, step = 92815 (45.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20822\n",
      "INFO:tensorflow:loss = 53.040363, step = 92915 (45.285 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20383\n",
      "INFO:tensorflow:loss = 22.458626, step = 93015 (45.376 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20751\n",
      "INFO:tensorflow:loss = 77.60981, step = 93115 (45.300 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20366\n",
      "INFO:tensorflow:loss = 47.47667, step = 93215 (45.379 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20347\n",
      "INFO:tensorflow:loss = 85.016045, step = 93315 (45.383 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 93357 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 43.07058.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-93357\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.66it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49897352283582513\n",
      "[test] Raw Hits@1: 37.64\n",
      "[test] Raw Hits@3: 57.14\n",
      "[test] Raw Hits@5: 64.4\n",
      "[test] Raw Hits@10: 73.38\n",
      "[test] Filtered MRR: 0.6679563070078575\n",
      "[test] Filtered Hits@1: 57.720000000000006\n",
      "[test] Filtered Hits@3: 72.61999999999999\n",
      "[test] Filtered Hits@5: 77.60000000000001\n",
      "[test] Filtered Hits@10: 82.94\n",
      "INFO:tensorflow:Epoch 100\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:\n",
      "[<tf.Variable 'e_embed:0' shape=(40943, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'p_embed:0' shape=(18, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/gamma:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization/beta:0' shape=(1,) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32_ref>,\n",
      " <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/gamma:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_1/beta:0' shape=(10368,) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/kernel:0' shape=(10368, 200) dtype=float32_ref>,\n",
      " <tf.Variable 'dense/bias:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/gamma:0' shape=(200,) dtype=float32_ref>,\n",
      " <tf.Variable 'batch_normalization_2/beta:0' shape=(200,) dtype=float32_ref>]\n",
      "INFO:tensorflow:params: 10287458\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-93357\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 93358 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:loss = 71.22413, step = 93358\n",
      "INFO:tensorflow:global_step/sec: 2.18954\n",
      "INFO:tensorflow:loss = 85.61457, step = 93458 (45.673 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20666\n",
      "INFO:tensorflow:loss = 68.947426, step = 93558 (45.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20237\n",
      "INFO:tensorflow:loss = 29.150646, step = 93658 (45.406 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20612\n",
      "INFO:tensorflow:loss = 29.456692, step = 93758 (45.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20806\n",
      "INFO:tensorflow:loss = 51.579483, step = 93858 (45.289 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.21445\n",
      "INFO:tensorflow:loss = 60.705467, step = 93958 (45.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20796\n",
      "INFO:tensorflow:loss = 35.387604, step = 94058 (45.291 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.21058\n",
      "INFO:tensorflow:loss = 57.153996, step = 94158 (45.237 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.20365\n",
      "INFO:tensorflow:loss = 28.458538, step = 94258 (45.380 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 94300 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 41.031498.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpj1bm9lou/model.ckpt-94300\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████| 5000/5000 [00:16<00:00, 304.60it/s]\n",
      "100%|█████████████████████████████| 5000/5000 [01:40<00:00, 49.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test] Raw MRR: 0.49699243419396805\n",
      "[test] Raw Hits@1: 37.36\n",
      "[test] Raw Hits@3: 57.14\n",
      "[test] Raw Hits@5: 64.60000000000001\n",
      "[test] Raw Hits@10: 73.38\n",
      "[test] Filtered MRR: 0.6681777772533513\n",
      "[test] Filtered Hits@1: 57.68\n",
      "[test] Filtered Hits@3: 72.86\n",
      "[test] Filtered Hits@5: 77.66\n",
      "[test] Filtered Hits@10: 83.04\n"
     ]
    }
   ],
   "source": [
    "triples_all, triples_tr, triples_va, triples_te = load_triple()\n",
    "e2idx, p2idx, params = build_vocab(triples_all)\n",
    "sp2o = build_multi_label(triples_tr)\n",
    "\n",
    "model = tf.estimator.Estimator(model_fn,\n",
    "                               params = params)\n",
    "\n",
    "for n_epoch in range(Config.n_epochs):\n",
    "    tf.logging.info('Epoch %d'%(n_epoch+1))\n",
    "    model.train(lambda: train_input_fn(triples_tr, e2idx, p2idx, sp2o))\n",
    "    evaluate_rank(model,\n",
    "                  triples_va,\n",
    "                  triples_te,\n",
    "                  triples_all,\n",
    "                  e2idx,\n",
    "                  p2idx,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
